lsst.ip.isr  22.0.1-14-g9f5bc66+6fc1e75841
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
30 import lsst.pipe.base.connectionTypes as cT
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36  ReadoutCorner)
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 
43 from . import isrFunctions
44 from . import isrQa
45 from . import linearize
46 from .defects import Defects
47 
48 from .assembleCcdTask import AssembleCcdTask
49 from .crosstalk import CrosstalkTask, CrosstalkCalib
50 from .fringe import FringeTask
51 from .isr import maskNans
52 from .masking import MaskingTask
53 from .overscan import OverscanCorrectionTask
54 from .straylight import StrayLightTask
55 from .vignette import VignetteTask
56 from lsst.daf.butler import DimensionGraph
57 
58 
59 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
60 
61 
62 def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections):
63  """Lookup function to identify crosstalkSource entries.
64 
65  This should return an empty list under most circumstances. Only
66  when inter-chip crosstalk has been identified should this be
67  populated.
68 
69  This will be unused until DM-25348 resolves the quantum graph
70  generation issue.
71 
72  Parameters
73  ----------
74  datasetType : `str`
75  Dataset to lookup.
76  registry : `lsst.daf.butler.Registry`
77  Butler registry to query.
78  quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79  Data id to transform to identify crosstalkSources. The
80  ``detector`` entry will be stripped.
81  collections : `lsst.daf.butler.CollectionSearch`
82  Collections to search through.
83 
84  Returns
85  -------
86  results : `list` [`lsst.daf.butler.DatasetRef`]
87  List of datasets that match the query that will be used as
88  crosstalkSources.
89  """
90  newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"]))
91  results = list(registry.queryDatasets(datasetType,
92  collections=collections,
93  dataId=newDataId,
94  findFirst=True,
95  ).expanded())
96  return results
97 
98 
99 class IsrTaskConnections(pipeBase.PipelineTaskConnections,
100  dimensions={"instrument", "exposure", "detector"},
101  defaultTemplates={}):
102  ccdExposure = cT.Input(
103  name="raw",
104  doc="Input exposure to process.",
105  storageClass="Exposure",
106  dimensions=["instrument", "exposure", "detector"],
107  )
108  camera = cT.PrerequisiteInput(
109  name="camera",
110  storageClass="Camera",
111  doc="Input camera to construct complete exposures.",
112  dimensions=["instrument"],
113  isCalibration=True,
114  )
115 
116  crosstalk = cT.PrerequisiteInput(
117  name="crosstalk",
118  doc="Input crosstalk object",
119  storageClass="CrosstalkCalib",
120  dimensions=["instrument", "detector"],
121  isCalibration=True,
122  minimum=0, # can fall back to cameraGeom
123  )
124  # TODO: DM-25348. This does not work yet to correctly load
125  # possible crosstalk sources.
126  crosstalkSources = cT.PrerequisiteInput(
127  name="isrOverscanCorrected",
128  doc="Overscan corrected input images.",
129  storageClass="Exposure",
130  dimensions=["instrument", "exposure", "detector"],
131  deferLoad=True,
132  multiple=True,
133  lookupFunction=crosstalkSourceLookup,
134  minimum=0, # not needed for all instruments, no config to control this
135  )
136  bias = cT.PrerequisiteInput(
137  name="bias",
138  doc="Input bias calibration.",
139  storageClass="ExposureF",
140  dimensions=["instrument", "detector"],
141  isCalibration=True,
142  )
143  dark = cT.PrerequisiteInput(
144  name='dark',
145  doc="Input dark calibration.",
146  storageClass="ExposureF",
147  dimensions=["instrument", "detector"],
148  isCalibration=True,
149  )
150  flat = cT.PrerequisiteInput(
151  name="flat",
152  doc="Input flat calibration.",
153  storageClass="ExposureF",
154  dimensions=["instrument", "physical_filter", "detector"],
155  isCalibration=True,
156  )
157  ptc = cT.PrerequisiteInput(
158  name="ptc",
159  doc="Input Photon Transfer Curve dataset",
160  storageClass="PhotonTransferCurveDataset",
161  dimensions=["instrument", "detector"],
162  isCalibration=True,
163  )
164  fringes = cT.PrerequisiteInput(
165  name="fringe",
166  doc="Input fringe calibration.",
167  storageClass="ExposureF",
168  dimensions=["instrument", "physical_filter", "detector"],
169  isCalibration=True,
170  minimum=0, # only needed for some bands, even when enabled
171  )
172  strayLightData = cT.PrerequisiteInput(
173  name='yBackground',
174  doc="Input stray light calibration.",
175  storageClass="StrayLightData",
176  dimensions=["instrument", "physical_filter", "detector"],
177  isCalibration=True,
178  minimum=0, # only needed for some bands, even when enabled
179  )
180  bfKernel = cT.PrerequisiteInput(
181  name='bfKernel',
182  doc="Input brighter-fatter kernel.",
183  storageClass="NumpyArray",
184  dimensions=["instrument"],
185  isCalibration=True,
186  minimum=0, # can use either bfKernel or newBFKernel
187  )
188  newBFKernel = cT.PrerequisiteInput(
189  name='brighterFatterKernel',
190  doc="Newer complete kernel + gain solutions.",
191  storageClass="BrighterFatterKernel",
192  dimensions=["instrument", "detector"],
193  isCalibration=True,
194  minimum=0, # can use either bfKernel or newBFKernel
195  )
196  defects = cT.PrerequisiteInput(
197  name='defects',
198  doc="Input defect tables.",
199  storageClass="Defects",
200  dimensions=["instrument", "detector"],
201  isCalibration=True,
202  )
203  linearizer = cT.PrerequisiteInput(
204  name='linearizer',
205  storageClass="Linearizer",
206  doc="Linearity correction calibration.",
207  dimensions=["instrument", "detector"],
208  isCalibration=True,
209  minimum=0, # can fall back to cameraGeom
210  )
211  opticsTransmission = cT.PrerequisiteInput(
212  name="transmission_optics",
213  storageClass="TransmissionCurve",
214  doc="Transmission curve due to the optics.",
215  dimensions=["instrument"],
216  isCalibration=True,
217  )
218  filterTransmission = cT.PrerequisiteInput(
219  name="transmission_filter",
220  storageClass="TransmissionCurve",
221  doc="Transmission curve due to the filter.",
222  dimensions=["instrument", "physical_filter"],
223  isCalibration=True,
224  )
225  sensorTransmission = cT.PrerequisiteInput(
226  name="transmission_sensor",
227  storageClass="TransmissionCurve",
228  doc="Transmission curve due to the sensor.",
229  dimensions=["instrument", "detector"],
230  isCalibration=True,
231  )
232  atmosphereTransmission = cT.PrerequisiteInput(
233  name="transmission_atmosphere",
234  storageClass="TransmissionCurve",
235  doc="Transmission curve due to the atmosphere.",
236  dimensions=["instrument"],
237  isCalibration=True,
238  )
239  illumMaskedImage = cT.PrerequisiteInput(
240  name="illum",
241  doc="Input illumination correction.",
242  storageClass="MaskedImageF",
243  dimensions=["instrument", "physical_filter", "detector"],
244  isCalibration=True,
245  )
246 
247  outputExposure = cT.Output(
248  name='postISRCCD',
249  doc="Output ISR processed exposure.",
250  storageClass="Exposure",
251  dimensions=["instrument", "exposure", "detector"],
252  )
253  preInterpExposure = cT.Output(
254  name='preInterpISRCCD',
255  doc="Output ISR processed exposure, with pixels left uninterpolated.",
256  storageClass="ExposureF",
257  dimensions=["instrument", "exposure", "detector"],
258  )
259  outputOssThumbnail = cT.Output(
260  name="OssThumb",
261  doc="Output Overscan-subtracted thumbnail image.",
262  storageClass="Thumbnail",
263  dimensions=["instrument", "exposure", "detector"],
264  )
265  outputFlattenedThumbnail = cT.Output(
266  name="FlattenedThumb",
267  doc="Output flat-corrected thumbnail image.",
268  storageClass="Thumbnail",
269  dimensions=["instrument", "exposure", "detector"],
270  )
271 
272  def __init__(self, *, config=None):
273  super().__init__(config=config)
274 
275  if config.doBias is not True:
276  self.prerequisiteInputs.discard("bias")
277  if config.doLinearize is not True:
278  self.prerequisiteInputs.discard("linearizer")
279  if config.doCrosstalk is not True:
280  self.inputs.discard("crosstalkSources")
281  self.prerequisiteInputs.discard("crosstalk")
282  if config.doBrighterFatter is not True:
283  self.prerequisiteInputs.discard("bfKernel")
284  self.prerequisiteInputs.discard("newBFKernel")
285  if config.doDefect is not True:
286  self.prerequisiteInputs.discard("defects")
287  if config.doDark is not True:
288  self.prerequisiteInputs.discard("dark")
289  if config.doFlat is not True:
290  self.prerequisiteInputs.discard("flat")
291  if config.doFringe is not True:
292  self.prerequisiteInputs.discard("fringe")
293  if config.doStrayLight is not True:
294  self.prerequisiteInputs.discard("strayLightData")
295  if config.usePtcGains is not True and config.usePtcReadNoise is not True:
296  self.prerequisiteInputs.discard("ptc")
297  if config.doAttachTransmissionCurve is not True:
298  self.prerequisiteInputs.discard("opticsTransmission")
299  self.prerequisiteInputs.discard("filterTransmission")
300  self.prerequisiteInputs.discard("sensorTransmission")
301  self.prerequisiteInputs.discard("atmosphereTransmission")
302  if config.doUseOpticsTransmission is not True:
303  self.prerequisiteInputs.discard("opticsTransmission")
304  if config.doUseFilterTransmission is not True:
305  self.prerequisiteInputs.discard("filterTransmission")
306  if config.doUseSensorTransmission is not True:
307  self.prerequisiteInputs.discard("sensorTransmission")
308  if config.doUseAtmosphereTransmission is not True:
309  self.prerequisiteInputs.discard("atmosphereTransmission")
310  if config.doIlluminationCorrection is not True:
311  self.prerequisiteInputs.discard("illumMaskedImage")
312 
313  if config.doWrite is not True:
314  self.outputs.discard("outputExposure")
315  self.outputs.discard("preInterpExposure")
316  self.outputs.discard("outputFlattenedThumbnail")
317  self.outputs.discard("outputOssThumbnail")
318  if config.doSaveInterpPixels is not True:
319  self.outputs.discard("preInterpExposure")
320  if config.qa.doThumbnailOss is not True:
321  self.outputs.discard("outputOssThumbnail")
322  if config.qa.doThumbnailFlattened is not True:
323  self.outputs.discard("outputFlattenedThumbnail")
324 
325 
326 class IsrTaskConfig(pipeBase.PipelineTaskConfig,
327  pipelineConnections=IsrTaskConnections):
328  """Configuration parameters for IsrTask.
329 
330  Items are grouped in the order in which they are executed by the task.
331  """
332  datasetType = pexConfig.Field(
333  dtype=str,
334  doc="Dataset type for input data; users will typically leave this alone, "
335  "but camera-specific ISR tasks will override it",
336  default="raw",
337  )
338 
339  fallbackFilterName = pexConfig.Field(
340  dtype=str,
341  doc="Fallback default filter name for calibrations.",
342  optional=True
343  )
344  useFallbackDate = pexConfig.Field(
345  dtype=bool,
346  doc="Pass observation date when using fallback filter.",
347  default=False,
348  )
349  expectWcs = pexConfig.Field(
350  dtype=bool,
351  default=True,
352  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
353  )
354  fwhm = pexConfig.Field(
355  dtype=float,
356  doc="FWHM of PSF in arcseconds.",
357  default=1.0,
358  )
359  qa = pexConfig.ConfigField(
360  dtype=isrQa.IsrQaConfig,
361  doc="QA related configuration options.",
362  )
363 
364  # Image conversion configuration
365  doConvertIntToFloat = pexConfig.Field(
366  dtype=bool,
367  doc="Convert integer raw images to floating point values?",
368  default=True,
369  )
370 
371  # Saturated pixel handling.
372  doSaturation = pexConfig.Field(
373  dtype=bool,
374  doc="Mask saturated pixels? NB: this is totally independent of the"
375  " interpolation option - this is ONLY setting the bits in the mask."
376  " To have them interpolated make sure doSaturationInterpolation=True",
377  default=True,
378  )
379  saturatedMaskName = pexConfig.Field(
380  dtype=str,
381  doc="Name of mask plane to use in saturation detection and interpolation",
382  default="SAT",
383  )
384  saturation = pexConfig.Field(
385  dtype=float,
386  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
387  default=float("NaN"),
388  )
389  growSaturationFootprintSize = pexConfig.Field(
390  dtype=int,
391  doc="Number of pixels by which to grow the saturation footprints",
392  default=1,
393  )
394 
395  # Suspect pixel handling.
396  doSuspect = pexConfig.Field(
397  dtype=bool,
398  doc="Mask suspect pixels?",
399  default=False,
400  )
401  suspectMaskName = pexConfig.Field(
402  dtype=str,
403  doc="Name of mask plane to use for suspect pixels",
404  default="SUSPECT",
405  )
406  numEdgeSuspect = pexConfig.Field(
407  dtype=int,
408  doc="Number of edge pixels to be flagged as untrustworthy.",
409  default=0,
410  )
411  edgeMaskLevel = pexConfig.ChoiceField(
412  dtype=str,
413  doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
414  default="DETECTOR",
415  allowed={
416  'DETECTOR': 'Mask only the edges of the full detector.',
417  'AMP': 'Mask edges of each amplifier.',
418  },
419  )
420 
421  # Initial masking options.
422  doSetBadRegions = pexConfig.Field(
423  dtype=bool,
424  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
425  default=True,
426  )
427  badStatistic = pexConfig.ChoiceField(
428  dtype=str,
429  doc="How to estimate the average value for BAD regions.",
430  default='MEANCLIP',
431  allowed={
432  "MEANCLIP": "Correct using the (clipped) mean of good data",
433  "MEDIAN": "Correct using the median of the good data",
434  },
435  )
436 
437  # Overscan subtraction configuration.
438  doOverscan = pexConfig.Field(
439  dtype=bool,
440  doc="Do overscan subtraction?",
441  default=True,
442  )
443  overscan = pexConfig.ConfigurableField(
444  target=OverscanCorrectionTask,
445  doc="Overscan subtraction task for image segments.",
446  )
447 
448  overscanFitType = pexConfig.ChoiceField(
449  dtype=str,
450  doc="The method for fitting the overscan bias level.",
451  default='MEDIAN',
452  allowed={
453  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
454  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
455  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
456  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
457  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
458  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
459  "MEAN": "Correct using the mean of the overscan region",
460  "MEANCLIP": "Correct using a clipped mean of the overscan region",
461  "MEDIAN": "Correct using the median of the overscan region",
462  "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
463  },
464  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
465  " This option will no longer be used, and will be removed after v20.")
466  )
467  overscanOrder = pexConfig.Field(
468  dtype=int,
469  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, "
470  "or number of spline knots if overscan fit type is a spline."),
471  default=1,
472  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
473  " This option will no longer be used, and will be removed after v20.")
474  )
475  overscanNumSigmaClip = pexConfig.Field(
476  dtype=float,
477  doc="Rejection threshold (sigma) for collapsing overscan before fit",
478  default=3.0,
479  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
480  " This option will no longer be used, and will be removed after v20.")
481  )
482  overscanIsInt = pexConfig.Field(
483  dtype=bool,
484  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
485  " and overscan.FitType=MEDIAN_PER_ROW.",
486  default=True,
487  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
488  " This option will no longer be used, and will be removed after v20.")
489  )
490  # These options do not get deprecated, as they define how we slice up the image data.
491  overscanNumLeadingColumnsToSkip = pexConfig.Field(
492  dtype=int,
493  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
494  default=0,
495  )
496  overscanNumTrailingColumnsToSkip = pexConfig.Field(
497  dtype=int,
498  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
499  default=0,
500  )
501  overscanMaxDev = pexConfig.Field(
502  dtype=float,
503  doc="Maximum deviation from the median for overscan",
504  default=1000.0, check=lambda x: x > 0
505  )
506  overscanBiasJump = pexConfig.Field(
507  dtype=bool,
508  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
509  default=False,
510  )
511  overscanBiasJumpKeyword = pexConfig.Field(
512  dtype=str,
513  doc="Header keyword containing information about devices.",
514  default="NO_SUCH_KEY",
515  )
516  overscanBiasJumpDevices = pexConfig.ListField(
517  dtype=str,
518  doc="List of devices that need piecewise overscan correction.",
519  default=(),
520  )
521  overscanBiasJumpLocation = pexConfig.Field(
522  dtype=int,
523  doc="Location of bias jump along y-axis.",
524  default=0,
525  )
526 
527  # Amplifier to CCD assembly configuration
528  doAssembleCcd = pexConfig.Field(
529  dtype=bool,
530  default=True,
531  doc="Assemble amp-level exposures into a ccd-level exposure?"
532  )
533  assembleCcd = pexConfig.ConfigurableField(
534  target=AssembleCcdTask,
535  doc="CCD assembly task",
536  )
537 
538  # General calibration configuration.
539  doAssembleIsrExposures = pexConfig.Field(
540  dtype=bool,
541  default=False,
542  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
543  )
544  doTrimToMatchCalib = pexConfig.Field(
545  dtype=bool,
546  default=False,
547  doc="Trim raw data to match calibration bounding boxes?"
548  )
549 
550  # Bias subtraction.
551  doBias = pexConfig.Field(
552  dtype=bool,
553  doc="Apply bias frame correction?",
554  default=True,
555  )
556  biasDataProductName = pexConfig.Field(
557  dtype=str,
558  doc="Name of the bias data product",
559  default="bias",
560  )
561  doBiasBeforeOverscan = pexConfig.Field(
562  dtype=bool,
563  doc="Reverse order of overscan and bias correction.",
564  default=False
565  )
566 
567  # Variance construction
568  doVariance = pexConfig.Field(
569  dtype=bool,
570  doc="Calculate variance?",
571  default=True
572  )
573  gain = pexConfig.Field(
574  dtype=float,
575  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
576  default=float("NaN"),
577  )
578  readNoise = pexConfig.Field(
579  dtype=float,
580  doc="The read noise to use if no Detector is present in the Exposure",
581  default=0.0,
582  )
583  doEmpiricalReadNoise = pexConfig.Field(
584  dtype=bool,
585  default=False,
586  doc="Calculate empirical read noise instead of value from AmpInfo data?"
587  )
588  usePtcReadNoise = pexConfig.Field(
589  dtype=bool,
590  default=False,
591  doc="Use readnoise values from the Photon Transfer Curve?"
592  )
593  maskNegativeVariance = pexConfig.Field(
594  dtype=bool,
595  default=True,
596  doc="Mask pixels that claim a negative variance? This likely indicates a failure "
597  "in the measurement of the overscan at an edge due to the data falling off faster "
598  "than the overscan model can account for it."
599  )
600  negativeVarianceMaskName = pexConfig.Field(
601  dtype=str,
602  default="BAD",
603  doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
604  )
605  # Linearization.
606  doLinearize = pexConfig.Field(
607  dtype=bool,
608  doc="Correct for nonlinearity of the detector's response?",
609  default=True,
610  )
611 
612  # Crosstalk.
613  doCrosstalk = pexConfig.Field(
614  dtype=bool,
615  doc="Apply intra-CCD crosstalk correction?",
616  default=False,
617  )
618  doCrosstalkBeforeAssemble = pexConfig.Field(
619  dtype=bool,
620  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
621  default=False,
622  )
623  crosstalk = pexConfig.ConfigurableField(
624  target=CrosstalkTask,
625  doc="Intra-CCD crosstalk correction",
626  )
627 
628  # Masking options.
629  doDefect = pexConfig.Field(
630  dtype=bool,
631  doc="Apply correction for CCD defects, e.g. hot pixels?",
632  default=True,
633  )
634  doNanMasking = pexConfig.Field(
635  dtype=bool,
636  doc="Mask non-finite (NAN, inf) pixels?",
637  default=True,
638  )
639  doWidenSaturationTrails = pexConfig.Field(
640  dtype=bool,
641  doc="Widen bleed trails based on their width?",
642  default=True
643  )
644 
645  # Brighter-Fatter correction.
646  doBrighterFatter = pexConfig.Field(
647  dtype=bool,
648  default=False,
649  doc="Apply the brighter-fatter correction?"
650  )
651  brighterFatterLevel = pexConfig.ChoiceField(
652  dtype=str,
653  default="DETECTOR",
654  doc="The level at which to correct for brighter-fatter.",
655  allowed={
656  "AMP": "Every amplifier treated separately.",
657  "DETECTOR": "One kernel per detector",
658  }
659  )
660  brighterFatterMaxIter = pexConfig.Field(
661  dtype=int,
662  default=10,
663  doc="Maximum number of iterations for the brighter-fatter correction"
664  )
665  brighterFatterThreshold = pexConfig.Field(
666  dtype=float,
667  default=1000,
668  doc="Threshold used to stop iterating the brighter-fatter correction. It is the "
669  "absolute value of the difference between the current corrected image and the one "
670  "from the previous iteration summed over all the pixels."
671  )
672  brighterFatterApplyGain = pexConfig.Field(
673  dtype=bool,
674  default=True,
675  doc="Should the gain be applied when applying the brighter-fatter correction?"
676  )
677  brighterFatterMaskListToInterpolate = pexConfig.ListField(
678  dtype=str,
679  doc="List of mask planes that should be interpolated over when applying the brighter-fatter "
680  "correction.",
681  default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"],
682  )
683  brighterFatterMaskGrowSize = pexConfig.Field(
684  dtype=int,
685  default=0,
686  doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
687  "when brighter-fatter correction is applied."
688  )
689 
690  # Dark subtraction.
691  doDark = pexConfig.Field(
692  dtype=bool,
693  doc="Apply dark frame correction?",
694  default=True,
695  )
696  darkDataProductName = pexConfig.Field(
697  dtype=str,
698  doc="Name of the dark data product",
699  default="dark",
700  )
701 
702  # Camera-specific stray light removal.
703  doStrayLight = pexConfig.Field(
704  dtype=bool,
705  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
706  default=False,
707  )
708  strayLight = pexConfig.ConfigurableField(
709  target=StrayLightTask,
710  doc="y-band stray light correction"
711  )
712 
713  # Flat correction.
714  doFlat = pexConfig.Field(
715  dtype=bool,
716  doc="Apply flat field correction?",
717  default=True,
718  )
719  flatDataProductName = pexConfig.Field(
720  dtype=str,
721  doc="Name of the flat data product",
722  default="flat",
723  )
724  flatScalingType = pexConfig.ChoiceField(
725  dtype=str,
726  doc="The method for scaling the flat on the fly.",
727  default='USER',
728  allowed={
729  "USER": "Scale by flatUserScale",
730  "MEAN": "Scale by the inverse of the mean",
731  "MEDIAN": "Scale by the inverse of the median",
732  },
733  )
734  flatUserScale = pexConfig.Field(
735  dtype=float,
736  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
737  default=1.0,
738  )
739  doTweakFlat = pexConfig.Field(
740  dtype=bool,
741  doc="Tweak flats to match observed amplifier ratios?",
742  default=False
743  )
744 
745  # Amplifier normalization based on gains instead of using flats configuration.
746  doApplyGains = pexConfig.Field(
747  dtype=bool,
748  doc="Correct the amplifiers for their gains instead of applying flat correction",
749  default=False,
750  )
751  usePtcGains = pexConfig.Field(
752  dtype=bool,
753  doc="Use the gain values from the Photon Transfer Curve?",
754  default=False,
755  )
756  normalizeGains = pexConfig.Field(
757  dtype=bool,
758  doc="Normalize all the amplifiers in each CCD to have the same median value.",
759  default=False,
760  )
761 
762  # Fringe correction.
763  doFringe = pexConfig.Field(
764  dtype=bool,
765  doc="Apply fringe correction?",
766  default=True,
767  )
768  fringe = pexConfig.ConfigurableField(
769  target=FringeTask,
770  doc="Fringe subtraction task",
771  )
772  fringeAfterFlat = pexConfig.Field(
773  dtype=bool,
774  doc="Do fringe subtraction after flat-fielding?",
775  default=True,
776  )
777 
778  # Initial CCD-level background statistics options.
779  doMeasureBackground = pexConfig.Field(
780  dtype=bool,
781  doc="Measure the background level on the reduced image?",
782  default=False,
783  )
784 
785  # Camera-specific masking configuration.
786  doCameraSpecificMasking = pexConfig.Field(
787  dtype=bool,
788  doc="Mask camera-specific bad regions?",
789  default=False,
790  )
791  masking = pexConfig.ConfigurableField(
792  target=MaskingTask,
793  doc="Masking task."
794  )
795 
796  # Interpolation options.
797 
798  doInterpolate = pexConfig.Field(
799  dtype=bool,
800  doc="Interpolate masked pixels?",
801  default=True,
802  )
803  doSaturationInterpolation = pexConfig.Field(
804  dtype=bool,
805  doc="Perform interpolation over pixels masked as saturated?"
806  " NB: This is independent of doSaturation; if that is False this plane"
807  " will likely be blank, resulting in a no-op here.",
808  default=True,
809  )
810  doNanInterpolation = pexConfig.Field(
811  dtype=bool,
812  doc="Perform interpolation over pixels masked as NaN?"
813  " NB: This is independent of doNanMasking; if that is False this plane"
814  " will likely be blank, resulting in a no-op here.",
815  default=True,
816  )
817  doNanInterpAfterFlat = pexConfig.Field(
818  dtype=bool,
819  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
820  "also have to interpolate them before flat-fielding."),
821  default=False,
822  )
823  maskListToInterpolate = pexConfig.ListField(
824  dtype=str,
825  doc="List of mask planes that should be interpolated.",
826  default=['SAT', 'BAD'],
827  )
828  doSaveInterpPixels = pexConfig.Field(
829  dtype=bool,
830  doc="Save a copy of the pre-interpolated pixel values?",
831  default=False,
832  )
833 
834  # Default photometric calibration options.
835  fluxMag0T1 = pexConfig.DictField(
836  keytype=str,
837  itemtype=float,
838  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
839  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
840  ))
841  )
842  defaultFluxMag0T1 = pexConfig.Field(
843  dtype=float,
844  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
845  default=pow(10.0, 0.4*28.0)
846  )
847 
848  # Vignette correction configuration.
849  doVignette = pexConfig.Field(
850  dtype=bool,
851  doc="Apply vignetting parameters?",
852  default=False,
853  )
854  vignette = pexConfig.ConfigurableField(
855  target=VignetteTask,
856  doc="Vignetting task.",
857  )
858 
859  # Transmission curve configuration.
860  doAttachTransmissionCurve = pexConfig.Field(
861  dtype=bool,
862  default=False,
863  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
864  )
865  doUseOpticsTransmission = pexConfig.Field(
866  dtype=bool,
867  default=True,
868  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
869  )
870  doUseFilterTransmission = pexConfig.Field(
871  dtype=bool,
872  default=True,
873  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
874  )
875  doUseSensorTransmission = pexConfig.Field(
876  dtype=bool,
877  default=True,
878  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
879  )
880  doUseAtmosphereTransmission = pexConfig.Field(
881  dtype=bool,
882  default=True,
883  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
884  )
885 
886  # Illumination correction.
887  doIlluminationCorrection = pexConfig.Field(
888  dtype=bool,
889  default=False,
890  doc="Perform illumination correction?"
891  )
892  illuminationCorrectionDataProductName = pexConfig.Field(
893  dtype=str,
894  doc="Name of the illumination correction data product.",
895  default="illumcor",
896  )
897  illumScale = pexConfig.Field(
898  dtype=float,
899  doc="Scale factor for the illumination correction.",
900  default=1.0,
901  )
902  illumFilters = pexConfig.ListField(
903  dtype=str,
904  default=[],
905  doc="Only perform illumination correction for these filters."
906  )
907 
908  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
909  doWrite = pexConfig.Field(
910  dtype=bool,
911  doc="Persist postISRCCD?",
912  default=True,
913  )
914 
915  def validate(self):
916  super().validate()
917  if self.doFlatdoFlat and self.doApplyGainsdoApplyGains:
918  raise ValueError("You may not specify both doFlat and doApplyGains")
919  if self.doBiasBeforeOverscandoBiasBeforeOverscan and self.doTrimToMatchCalibdoTrimToMatchCalib:
920  raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
921  if self.doSaturationInterpolationdoSaturationInterpolation and self.saturatedMaskNamesaturatedMaskName not in self.maskListToInterpolatemaskListToInterpolate:
922  self.maskListToInterpolatemaskListToInterpolate.append(self.saturatedMaskNamesaturatedMaskName)
923  if not self.doSaturationInterpolationdoSaturationInterpolation and self.saturatedMaskNamesaturatedMaskName in self.maskListToInterpolatemaskListToInterpolate:
924  self.maskListToInterpolatemaskListToInterpolate.remove(self.saturatedMaskNamesaturatedMaskName)
925  if self.doNanInterpolationdoNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolatemaskListToInterpolate:
926  self.maskListToInterpolatemaskListToInterpolate.append("UNMASKEDNAN")
927 
928 
929 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
930  """Apply common instrument signature correction algorithms to a raw frame.
931 
932  The process for correcting imaging data is very similar from
933  camera to camera. This task provides a vanilla implementation of
934  doing these corrections, including the ability to turn certain
935  corrections off if they are not needed. The inputs to the primary
936  method, `run()`, are a raw exposure to be corrected and the
937  calibration data products. The raw input is a single chip sized
938  mosaic of all amps including overscans and other non-science
939  pixels. The method `runDataRef()` identifies and defines the
940  calibration data products, and is intended for use by a
941  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
942  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
943  subclassed for different camera, although the most camera specific
944  methods have been split into subtasks that can be redirected
945  appropriately.
946 
947  The __init__ method sets up the subtasks for ISR processing, using
948  the defaults from `lsst.ip.isr`.
949 
950  Parameters
951  ----------
952  args : `list`
953  Positional arguments passed to the Task constructor. None used at this time.
954  kwargs : `dict`, optional
955  Keyword arguments passed on to the Task constructor. None used at this time.
956  """
957  ConfigClass = IsrTaskConfig
958  _DefaultName = "isr"
959 
960  def __init__(self, **kwargs):
961  super().__init__(**kwargs)
962  self.makeSubtask("assembleCcd")
963  self.makeSubtask("crosstalk")
964  self.makeSubtask("strayLight")
965  self.makeSubtask("fringe")
966  self.makeSubtask("masking")
967  self.makeSubtask("overscan")
968  self.makeSubtask("vignette")
969 
970  def runQuantum(self, butlerQC, inputRefs, outputRefs):
971  inputs = butlerQC.get(inputRefs)
972 
973  try:
974  inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
975  except Exception as e:
976  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
977  (inputRefs, e))
978 
979  inputs['isGen3'] = True
980 
981  detector = inputs['ccdExposure'].getDetector()
982 
983  if self.config.doCrosstalk is True:
984  # Crosstalk sources need to be defined by the pipeline
985  # yaml if they exist.
986  if 'crosstalk' in inputs and inputs['crosstalk'] is not None:
987  if not isinstance(inputs['crosstalk'], CrosstalkCalib):
988  inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk'])
989  else:
990  coeffVector = (self.config.crosstalk.crosstalkValues
991  if self.config.crosstalk.useConfigCoefficients else None)
992  crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
993  inputs['crosstalk'] = crosstalkCalib
994  if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0:
995  if 'crosstalkSources' not in inputs:
996  self.log.warning("No crosstalkSources found for chip with interChip terms!")
997 
998  if self.doLinearizedoLinearize(detector) is True:
999  if 'linearizer' in inputs:
1000  if isinstance(inputs['linearizer'], dict):
1001  linearizer = linearize.Linearizer(detector=detector, log=self.log)
1002  linearizer.fromYaml(inputs['linearizer'])
1003  self.log.warning("Dictionary linearizers will be deprecated in DM-28741.")
1004  elif isinstance(inputs['linearizer'], numpy.ndarray):
1005  linearizer = linearize.Linearizer(table=inputs.get('linearizer', None),
1006  detector=detector,
1007  log=self.log)
1008  self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.")
1009  else:
1010  linearizer = inputs['linearizer']
1011  linearizer.log = self.log
1012  inputs['linearizer'] = linearizer
1013  else:
1014  inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log)
1015  self.log.warning("Constructing linearizer from cameraGeom information.")
1016 
1017  if self.config.doDefect is True:
1018  if "defects" in inputs and inputs['defects'] is not None:
1019  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
1020  # masking expects a list of defects defined by their bounding box
1021  if not isinstance(inputs["defects"], Defects):
1022  inputs["defects"] = Defects.fromTable(inputs["defects"])
1023 
1024  # Load the correct style of brighter-fatter kernel, and repack
1025  # the information as a numpy array.
1026  if self.config.doBrighterFatter:
1027  brighterFatterKernel = inputs.pop('newBFKernel', None)
1028  if brighterFatterKernel is None:
1029  brighterFatterKernel = inputs.get('bfKernel', None)
1030 
1031  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1032  # This is a ISR calib kernel
1033  detName = detector.getName()
1034  level = brighterFatterKernel.level
1035 
1036  # This is expected to be a dictionary of amp-wise gains.
1037  inputs['bfGains'] = brighterFatterKernel.gain
1038  if self.config.brighterFatterLevel == 'DETECTOR':
1039  if level == 'DETECTOR':
1040  if detName in brighterFatterKernel.detKernels:
1041  inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1042  else:
1043  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1044  elif level == 'AMP':
1045  self.log.warning("Making DETECTOR level kernel from AMP based brighter "
1046  "fatter kernels.")
1047  brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1048  inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1049  elif self.config.brighterFatterLevel == 'AMP':
1050  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1051 
1052  if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
1053  expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
1054  inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
1055  expId=expId,
1056  assembler=self.assembleCcd
1057  if self.config.doAssembleIsrExposures else None)
1058  else:
1059  inputs['fringes'] = pipeBase.Struct(fringes=None)
1060 
1061  if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
1062  if 'strayLightData' not in inputs:
1063  inputs['strayLightData'] = None
1064 
1065  outputs = self.runrun(**inputs)
1066  butlerQC.put(outputs, outputRefs)
1067 
1068  def readIsrData(self, dataRef, rawExposure):
1069  """Retrieve necessary frames for instrument signature removal.
1070 
1071  Pre-fetching all required ISR data products limits the IO
1072  required by the ISR. Any conflict between the calibration data
1073  available and that needed for ISR is also detected prior to
1074  doing processing, allowing it to fail quickly.
1075 
1076  Parameters
1077  ----------
1078  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1079  Butler reference of the detector data to be processed
1080  rawExposure : `afw.image.Exposure`
1081  The raw exposure that will later be corrected with the
1082  retrieved calibration data; should not be modified in this
1083  method.
1084 
1085  Returns
1086  -------
1087  result : `lsst.pipe.base.Struct`
1088  Result struct with components (which may be `None`):
1089  - ``bias``: bias calibration frame (`afw.image.Exposure`)
1090  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
1091  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1092  - ``dark``: dark calibration frame (`afw.image.Exposure`)
1093  - ``flat``: flat calibration frame (`afw.image.Exposure`)
1094  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1095  - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1096  - ``fringes``: `lsst.pipe.base.Struct` with components:
1097  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1098  - ``seed``: random seed derived from the ccdExposureId for random
1099  number generator (`uint32`).
1100  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1101  A ``TransmissionCurve`` that represents the throughput of the optics,
1102  to be evaluated in focal-plane coordinates.
1103  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1104  A ``TransmissionCurve`` that represents the throughput of the filter
1105  itself, to be evaluated in focal-plane coordinates.
1106  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1107  A ``TransmissionCurve`` that represents the throughput of the sensor
1108  itself, to be evaluated in post-assembly trimmed detector coordinates.
1109  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1110  A ``TransmissionCurve`` that represents the throughput of the
1111  atmosphere, assumed to be spatially constant.
1112  - ``strayLightData`` : `object`
1113  An opaque object containing calibration information for
1114  stray-light correction. If `None`, no correction will be
1115  performed.
1116  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1117 
1118  Raises
1119  ------
1120  NotImplementedError :
1121  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1122  """
1123  try:
1124  dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1125  dateObs = dateObs.toPython().isoformat()
1126  except RuntimeError:
1127  self.log.warning("Unable to identify dateObs for rawExposure.")
1128  dateObs = None
1129 
1130  ccd = rawExposure.getDetector()
1131  filterLabel = rawExposure.getFilterLabel()
1132  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1133  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
1134  biasExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.biasDataProductName)
1135  if self.config.doBias else None)
1136  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
1137  linearizer = (dataRef.get("linearizer", immediate=True)
1138  if self.doLinearizedoLinearize(ccd) else None)
1139  if linearizer is not None and not isinstance(linearizer, numpy.ndarray):
1140  linearizer.log = self.log
1141  if isinstance(linearizer, numpy.ndarray):
1142  linearizer = linearize.Linearizer(table=linearizer, detector=ccd)
1143 
1144  crosstalkCalib = None
1145  if self.config.doCrosstalk:
1146  try:
1147  crosstalkCalib = dataRef.get("crosstalk", immediate=True)
1148  except NoResults:
1149  coeffVector = (self.config.crosstalk.crosstalkValues
1150  if self.config.crosstalk.useConfigCoefficients else None)
1151  crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1152  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1153  if self.config.doCrosstalk else None)
1154 
1155  darkExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.darkDataProductName)
1156  if self.config.doDark else None)
1157  flatExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.flatDataProductName,
1158  dateObs=dateObs)
1159  if self.config.doFlat else None)
1160 
1161  brighterFatterKernel = None
1162  brighterFatterGains = None
1163  if self.config.doBrighterFatter is True:
1164  try:
1165  # Use the new-style cp_pipe version of the kernel if it exists
1166  # If using a new-style kernel, always use the self-consistent
1167  # gains, i.e. the ones inside the kernel object itself
1168  brighterFatterKernel = dataRef.get("brighterFatterKernel")
1169  brighterFatterGains = brighterFatterKernel.gain
1170  self.log.info("New style brighter-fatter kernel (brighterFatterKernel) loaded")
1171  except NoResults:
1172  try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
1173  brighterFatterKernel = dataRef.get("bfKernel")
1174  self.log.info("Old style brighter-fatter kernel (bfKernel) loaded")
1175  except NoResults:
1176  brighterFatterKernel = None
1177  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1178  # If the kernel is not an ndarray, it's the cp_pipe version
1179  # so extract the kernel for this detector, or raise an error
1180  if self.config.brighterFatterLevel == 'DETECTOR':
1181  if brighterFatterKernel.detKernels:
1182  brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()]
1183  else:
1184  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1185  else:
1186  # TODO DM-15631 for implementing this
1187  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1188 
1189  defectList = (dataRef.get("defects")
1190  if self.config.doDefect else None)
1191  expId = rawExposure.getInfo().getVisitInfo().getExposureId()
1192  fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1193  if self.config.doAssembleIsrExposures else None)
1194  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1195  else pipeBase.Struct(fringes=None))
1196 
1197  if self.config.doAttachTransmissionCurve:
1198  opticsTransmission = (dataRef.get("transmission_optics")
1199  if self.config.doUseOpticsTransmission else None)
1200  filterTransmission = (dataRef.get("transmission_filter")
1201  if self.config.doUseFilterTransmission else None)
1202  sensorTransmission = (dataRef.get("transmission_sensor")
1203  if self.config.doUseSensorTransmission else None)
1204  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1205  if self.config.doUseAtmosphereTransmission else None)
1206  else:
1207  opticsTransmission = None
1208  filterTransmission = None
1209  sensorTransmission = None
1210  atmosphereTransmission = None
1211 
1212  if self.config.doStrayLight:
1213  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1214  else:
1215  strayLightData = None
1216 
1217  illumMaskedImage = (self.getIsrExposuregetIsrExposure(dataRef,
1218  self.config.illuminationCorrectionDataProductName).getMaskedImage()
1219  if (self.config.doIlluminationCorrection
1220  and physicalFilter in self.config.illumFilters)
1221  else None)
1222 
1223  # Struct should include only kwargs to run()
1224  return pipeBase.Struct(bias=biasExposure,
1225  linearizer=linearizer,
1226  crosstalk=crosstalkCalib,
1227  crosstalkSources=crosstalkSources,
1228  dark=darkExposure,
1229  flat=flatExposure,
1230  bfKernel=brighterFatterKernel,
1231  bfGains=brighterFatterGains,
1232  defects=defectList,
1233  fringes=fringeStruct,
1234  opticsTransmission=opticsTransmission,
1235  filterTransmission=filterTransmission,
1236  sensorTransmission=sensorTransmission,
1237  atmosphereTransmission=atmosphereTransmission,
1238  strayLightData=strayLightData,
1239  illumMaskedImage=illumMaskedImage
1240  )
1241 
1242  @pipeBase.timeMethod
1243  def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1244  crosstalk=None, crosstalkSources=None,
1245  dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1246  fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1247  sensorTransmission=None, atmosphereTransmission=None,
1248  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1249  isGen3=False,
1250  ):
1251  """Perform instrument signature removal on an exposure.
1252 
1253  Steps included in the ISR processing, in order performed, are:
1254  - saturation and suspect pixel masking
1255  - overscan subtraction
1256  - CCD assembly of individual amplifiers
1257  - bias subtraction
1258  - variance image construction
1259  - linearization of non-linear response
1260  - crosstalk masking
1261  - brighter-fatter correction
1262  - dark subtraction
1263  - fringe correction
1264  - stray light subtraction
1265  - flat correction
1266  - masking of known defects and camera specific features
1267  - vignette calculation
1268  - appending transmission curve and distortion model
1269 
1270  Parameters
1271  ----------
1272  ccdExposure : `lsst.afw.image.Exposure`
1273  The raw exposure that is to be run through ISR. The
1274  exposure is modified by this method.
1275  camera : `lsst.afw.cameraGeom.Camera`, optional
1276  The camera geometry for this exposure. Required if ``isGen3`` is
1277  `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1278  ``flat`` does not have an associated detector.
1279  bias : `lsst.afw.image.Exposure`, optional
1280  Bias calibration frame.
1281  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1282  Functor for linearization.
1283  crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1284  Calibration for crosstalk.
1285  crosstalkSources : `list`, optional
1286  List of possible crosstalk sources.
1287  dark : `lsst.afw.image.Exposure`, optional
1288  Dark calibration frame.
1289  flat : `lsst.afw.image.Exposure`, optional
1290  Flat calibration frame.
1291  ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
1292  Photon transfer curve dataset, with, e.g., gains
1293  and read noise.
1294  bfKernel : `numpy.ndarray`, optional
1295  Brighter-fatter kernel.
1296  bfGains : `dict` of `float`, optional
1297  Gains used to override the detector's nominal gains for the
1298  brighter-fatter correction. A dict keyed by amplifier name for
1299  the detector in question.
1300  defects : `lsst.ip.isr.Defects`, optional
1301  List of defects.
1302  fringes : `lsst.pipe.base.Struct`, optional
1303  Struct containing the fringe correction data, with
1304  elements:
1305  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1306  - ``seed``: random seed derived from the ccdExposureId for random
1307  number generator (`uint32`)
1308  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1309  A ``TransmissionCurve`` that represents the throughput of the optics,
1310  to be evaluated in focal-plane coordinates.
1311  filterTransmission : `lsst.afw.image.TransmissionCurve`
1312  A ``TransmissionCurve`` that represents the throughput of the filter
1313  itself, to be evaluated in focal-plane coordinates.
1314  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1315  A ``TransmissionCurve`` that represents the throughput of the sensor
1316  itself, to be evaluated in post-assembly trimmed detector coordinates.
1317  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1318  A ``TransmissionCurve`` that represents the throughput of the
1319  atmosphere, assumed to be spatially constant.
1320  detectorNum : `int`, optional
1321  The integer number for the detector to process.
1322  isGen3 : bool, optional
1323  Flag this call to run() as using the Gen3 butler environment.
1324  strayLightData : `object`, optional
1325  Opaque object containing calibration information for stray-light
1326  correction. If `None`, no correction will be performed.
1327  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1328  Illumination correction image.
1329 
1330  Returns
1331  -------
1332  result : `lsst.pipe.base.Struct`
1333  Result struct with component:
1334  - ``exposure`` : `afw.image.Exposure`
1335  The fully ISR corrected exposure.
1336  - ``outputExposure`` : `afw.image.Exposure`
1337  An alias for `exposure`
1338  - ``ossThumb`` : `numpy.ndarray`
1339  Thumbnail image of the exposure after overscan subtraction.
1340  - ``flattenedThumb`` : `numpy.ndarray`
1341  Thumbnail image of the exposure after flat-field correction.
1342 
1343  Raises
1344  ------
1345  RuntimeError
1346  Raised if a configuration option is set to True, but the
1347  required calibration data has not been specified.
1348 
1349  Notes
1350  -----
1351  The current processed exposure can be viewed by setting the
1352  appropriate lsstDebug entries in the `debug.display`
1353  dictionary. The names of these entries correspond to some of
1354  the IsrTaskConfig Boolean options, with the value denoting the
1355  frame to use. The exposure is shown inside the matching
1356  option check and after the processing of that step has
1357  finished. The steps with debug points are:
1358 
1359  doAssembleCcd
1360  doBias
1361  doCrosstalk
1362  doBrighterFatter
1363  doDark
1364  doFringe
1365  doStrayLight
1366  doFlat
1367 
1368  In addition, setting the "postISRCCD" entry displays the
1369  exposure after all ISR processing has finished.
1370 
1371  """
1372 
1373  if isGen3 is True:
1374  # Gen3 currently cannot automatically do configuration overrides.
1375  # DM-15257 looks to discuss this issue.
1376  # Configure input exposures;
1377  if detectorNum is None:
1378  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1379 
1380  ccdExposure = self.ensureExposureensureExposure(ccdExposure, camera, detectorNum)
1381  bias = self.ensureExposureensureExposure(bias, camera, detectorNum)
1382  dark = self.ensureExposureensureExposure(dark, camera, detectorNum)
1383  flat = self.ensureExposureensureExposure(flat, camera, detectorNum)
1384  else:
1385  if isinstance(ccdExposure, ButlerDataRef):
1386  return self.runDataRefrunDataRef(ccdExposure)
1387 
1388  ccd = ccdExposure.getDetector()
1389  filterLabel = ccdExposure.getFilterLabel()
1390  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1391 
1392  if not ccd:
1393  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1394  ccd = [FakeAmp(ccdExposure, self.config)]
1395 
1396  # Validate Input
1397  if self.config.doBias and bias is None:
1398  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1399  if self.doLinearizedoLinearize(ccd) and linearizer is None:
1400  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1401  if self.config.doBrighterFatter and bfKernel is None:
1402  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1403  if self.config.doDark and dark is None:
1404  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1405  if self.config.doFlat and flat is None:
1406  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1407  if self.config.doDefect and defects is None:
1408  raise RuntimeError("Must supply defects if config.doDefect=True.")
1409  if (self.config.doFringe and physicalFilter in self.fringe.config.filters
1410  and fringes.fringes is None):
1411  # The `fringes` object needs to be a pipeBase.Struct, as
1412  # we use it as a `dict` for the parameters of
1413  # `FringeTask.run()`. The `fringes.fringes` `list` may
1414  # not be `None` if `doFringe=True`. Otherwise, raise.
1415  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1416  if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters
1417  and illumMaskedImage is None):
1418  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1419 
1420  # Begin ISR processing.
1421  if self.config.doConvertIntToFloat:
1422  self.log.info("Converting exposure to floating point values.")
1423  ccdExposure = self.convertIntToFloatconvertIntToFloat(ccdExposure)
1424 
1425  if self.config.doBias and self.config.doBiasBeforeOverscan:
1426  self.log.info("Applying bias correction.")
1427  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1428  trimToFit=self.config.doTrimToMatchCalib)
1429  self.debugViewdebugView(ccdExposure, "doBias")
1430 
1431  # Amplifier level processing.
1432  overscans = []
1433  for amp in ccd:
1434  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1435  if ccdExposure.getBBox().contains(amp.getBBox()):
1436  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1437  badAmp = self.maskAmplifiermaskAmplifier(ccdExposure, amp, defects)
1438 
1439  if self.config.doOverscan and not badAmp:
1440  # Overscan correction on amp-by-amp basis.
1441  overscanResults = self.overscanCorrectionoverscanCorrection(ccdExposure, amp)
1442  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1443  if overscanResults is not None and \
1444  self.config.qa is not None and self.config.qa.saveStats is True:
1445  if isinstance(overscanResults.overscanFit, float):
1446  qaMedian = overscanResults.overscanFit
1447  qaStdev = float("NaN")
1448  else:
1449  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1450  afwMath.MEDIAN | afwMath.STDEVCLIP)
1451  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1452  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1453 
1454  self.metadata.set(f"FIT MEDIAN {amp.getName()}", qaMedian)
1455  self.metadata.set(f"FIT STDEV {amp.getName()}", qaStdev)
1456  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1457  amp.getName(), qaMedian, qaStdev)
1458 
1459  # Residuals after overscan correction
1460  qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1461  afwMath.MEDIAN | afwMath.STDEVCLIP)
1462  qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1463  qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1464 
1465  self.metadata.set(f"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter)
1466  self.metadata.set(f"RESIDUAL STDEV {amp.getName()}", qaStdevAfter)
1467  self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f",
1468  amp.getName(), qaMedianAfter, qaStdevAfter)
1469 
1470  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1471  else:
1472  if badAmp:
1473  self.log.warning("Amplifier %s is bad.", amp.getName())
1474  overscanResults = None
1475 
1476  overscans.append(overscanResults if overscanResults is not None else None)
1477  else:
1478  self.log.info("Skipped OSCAN for %s.", amp.getName())
1479 
1480  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1481  self.log.info("Applying crosstalk correction.")
1482  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1483  crosstalkSources=crosstalkSources, camera=camera)
1484  self.debugViewdebugView(ccdExposure, "doCrosstalk")
1485 
1486  if self.config.doAssembleCcd:
1487  self.log.info("Assembling CCD from amplifiers.")
1488  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1489 
1490  if self.config.expectWcs and not ccdExposure.getWcs():
1491  self.log.warning("No WCS found in input exposure.")
1492  self.debugViewdebugView(ccdExposure, "doAssembleCcd")
1493 
1494  ossThumb = None
1495  if self.config.qa.doThumbnailOss:
1496  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1497 
1498  if self.config.doBias and not self.config.doBiasBeforeOverscan:
1499  self.log.info("Applying bias correction.")
1500  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1501  trimToFit=self.config.doTrimToMatchCalib)
1502  self.debugViewdebugView(ccdExposure, "doBias")
1503 
1504  if self.config.doVariance:
1505  for amp, overscanResults in zip(ccd, overscans):
1506  if ccdExposure.getBBox().contains(amp.getBBox()):
1507  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1508  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1509  if overscanResults is not None:
1510  self.updateVarianceupdateVariance(ampExposure, amp,
1511  overscanImage=overscanResults.overscanImage,
1512  ptcDataset=ptc)
1513  else:
1514  self.updateVarianceupdateVariance(ampExposure, amp,
1515  overscanImage=None,
1516  ptcDataset=ptc)
1517  if self.config.qa is not None and self.config.qa.saveStats is True:
1518  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1519  afwMath.MEDIAN | afwMath.STDEVCLIP)
1520  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1521  qaStats.getValue(afwMath.MEDIAN))
1522  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1523  qaStats.getValue(afwMath.STDEVCLIP))
1524  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1525  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1526  qaStats.getValue(afwMath.STDEVCLIP))
1527  if self.config.maskNegativeVariance:
1528  self.maskNegativeVariancemaskNegativeVariance(ccdExposure)
1529 
1530  if self.doLinearizedoLinearize(ccd):
1531  self.log.info("Applying linearizer.")
1532  linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1533  detector=ccd, log=self.log)
1534 
1535  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1536  self.log.info("Applying crosstalk correction.")
1537  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1538  crosstalkSources=crosstalkSources, isTrimmed=True)
1539  self.debugViewdebugView(ccdExposure, "doCrosstalk")
1540 
1541  # Masking block. Optionally mask known defects, NAN/inf pixels, widen trails, and do
1542  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1543  if self.config.doDefect:
1544  self.log.info("Masking defects.")
1545  self.maskDefectmaskDefect(ccdExposure, defects)
1546 
1547  if self.config.numEdgeSuspect > 0:
1548  self.log.info("Masking edges as SUSPECT.")
1549  self.maskEdgesmaskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1550  maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
1551 
1552  if self.config.doNanMasking:
1553  self.log.info("Masking non-finite (NAN, inf) value pixels.")
1554  self.maskNanmaskNan(ccdExposure)
1555 
1556  if self.config.doWidenSaturationTrails:
1557  self.log.info("Widening saturation trails.")
1558  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1559 
1560  if self.config.doCameraSpecificMasking:
1561  self.log.info("Masking regions for camera specific reasons.")
1562  self.masking.run(ccdExposure)
1563 
1564  if self.config.doBrighterFatter:
1565  # We need to apply flats and darks before we can interpolate, and we
1566  # need to interpolate before we do B-F, but we do B-F without the
1567  # flats and darks applied so we can work in units of electrons or holes.
1568  # This context manager applies and then removes the darks and flats.
1569  #
1570  # We also do not want to interpolate values here, so operate on temporary
1571  # images so we can apply only the BF-correction and roll back the
1572  # interpolation.
1573  interpExp = ccdExposure.clone()
1574  with self.flatContextflatContext(interpExp, flat, dark):
1575  isrFunctions.interpolateFromMask(
1576  maskedImage=interpExp.getMaskedImage(),
1577  fwhm=self.config.fwhm,
1578  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1579  maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1580  )
1581  bfExp = interpExp.clone()
1582 
1583  self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.",
1584  type(bfKernel), type(bfGains))
1585  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1586  self.config.brighterFatterMaxIter,
1587  self.config.brighterFatterThreshold,
1588  self.config.brighterFatterApplyGain,
1589  bfGains)
1590  if bfResults[1] == self.config.brighterFatterMaxIter:
1591  self.log.warning("Brighter-fatter correction did not converge, final difference %f.",
1592  bfResults[0])
1593  else:
1594  self.log.info("Finished brighter-fatter correction in %d iterations.",
1595  bfResults[1])
1596  image = ccdExposure.getMaskedImage().getImage()
1597  bfCorr = bfExp.getMaskedImage().getImage()
1598  bfCorr -= interpExp.getMaskedImage().getImage()
1599  image += bfCorr
1600 
1601  # Applying the brighter-fatter correction applies a
1602  # convolution to the science image. At the edges this
1603  # convolution may not have sufficient valid pixels to
1604  # produce a valid correction. Mark pixels within the size
1605  # of the brighter-fatter kernel as EDGE to warn of this
1606  # fact.
1607  self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1608  self.maskEdgesmaskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1609  maskPlane="EDGE")
1610 
1611  if self.config.brighterFatterMaskGrowSize > 0:
1612  self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1613  for maskPlane in self.config.brighterFatterMaskListToInterpolate:
1614  isrFunctions.growMasks(ccdExposure.getMask(),
1615  radius=self.config.brighterFatterMaskGrowSize,
1616  maskNameList=maskPlane,
1617  maskValue=maskPlane)
1618 
1619  self.debugViewdebugView(ccdExposure, "doBrighterFatter")
1620 
1621  if self.config.doDark:
1622  self.log.info("Applying dark correction.")
1623  self.darkCorrectiondarkCorrection(ccdExposure, dark)
1624  self.debugViewdebugView(ccdExposure, "doDark")
1625 
1626  if self.config.doFringe and not self.config.fringeAfterFlat:
1627  self.log.info("Applying fringe correction before flat.")
1628  self.fringe.run(ccdExposure, **fringes.getDict())
1629  self.debugViewdebugView(ccdExposure, "doFringe")
1630 
1631  if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1632  self.log.info("Checking strayLight correction.")
1633  self.strayLight.run(ccdExposure, strayLightData)
1634  self.debugViewdebugView(ccdExposure, "doStrayLight")
1635 
1636  if self.config.doFlat:
1637  self.log.info("Applying flat correction.")
1638  self.flatCorrectionflatCorrection(ccdExposure, flat)
1639  self.debugViewdebugView(ccdExposure, "doFlat")
1640 
1641  if self.config.doApplyGains:
1642  self.log.info("Applying gain correction instead of flat.")
1643  if self.config.usePtcGains:
1644  self.log.info("Using gains from the Photon Transfer Curve.")
1645  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1646  ptcGains=ptc.gain)
1647  else:
1648  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1649 
1650  if self.config.doFringe and self.config.fringeAfterFlat:
1651  self.log.info("Applying fringe correction after flat.")
1652  self.fringe.run(ccdExposure, **fringes.getDict())
1653 
1654  if self.config.doVignette:
1655  self.log.info("Constructing Vignette polygon.")
1656  self.vignettePolygonvignettePolygon = self.vignette.run(ccdExposure)
1657 
1658  if self.config.vignette.doWriteVignettePolygon:
1659  self.setValidPolygonIntersectsetValidPolygonIntersect(ccdExposure, self.vignettePolygonvignettePolygon)
1660 
1661  if self.config.doAttachTransmissionCurve:
1662  self.log.info("Adding transmission curves.")
1663  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1664  filterTransmission=filterTransmission,
1665  sensorTransmission=sensorTransmission,
1666  atmosphereTransmission=atmosphereTransmission)
1667 
1668  flattenedThumb = None
1669  if self.config.qa.doThumbnailFlattened:
1670  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1671 
1672  if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters:
1673  self.log.info("Performing illumination correction.")
1674  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1675  illumMaskedImage, illumScale=self.config.illumScale,
1676  trimToFit=self.config.doTrimToMatchCalib)
1677 
1678  preInterpExp = None
1679  if self.config.doSaveInterpPixels:
1680  preInterpExp = ccdExposure.clone()
1681 
1682  # Reset and interpolate bad pixels.
1683  #
1684  # Large contiguous bad regions (which should have the BAD mask
1685  # bit set) should have their values set to the image median.
1686  # This group should include defects and bad amplifiers. As the
1687  # area covered by these defects are large, there's little
1688  # reason to expect that interpolation would provide a more
1689  # useful value.
1690  #
1691  # Smaller defects can be safely interpolated after the larger
1692  # regions have had their pixel values reset. This ensures
1693  # that the remaining defects adjacent to bad amplifiers (as an
1694  # example) do not attempt to interpolate extreme values.
1695  if self.config.doSetBadRegions:
1696  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1697  if badPixelCount > 0:
1698  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1699 
1700  if self.config.doInterpolate:
1701  self.log.info("Interpolating masked pixels.")
1702  isrFunctions.interpolateFromMask(
1703  maskedImage=ccdExposure.getMaskedImage(),
1704  fwhm=self.config.fwhm,
1705  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1706  maskNameList=list(self.config.maskListToInterpolate)
1707  )
1708 
1709  self.roughZeroPointroughZeroPoint(ccdExposure)
1710 
1711  if self.config.doMeasureBackground:
1712  self.log.info("Measuring background level.")
1713  self.measureBackgroundmeasureBackground(ccdExposure, self.config.qa)
1714 
1715  if self.config.qa is not None and self.config.qa.saveStats is True:
1716  for amp in ccd:
1717  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1718  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1719  afwMath.MEDIAN | afwMath.STDEVCLIP)
1720  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1721  qaStats.getValue(afwMath.MEDIAN))
1722  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1723  qaStats.getValue(afwMath.STDEVCLIP))
1724  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1725  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1726  qaStats.getValue(afwMath.STDEVCLIP))
1727 
1728  self.debugViewdebugView(ccdExposure, "postISRCCD")
1729 
1730  return pipeBase.Struct(
1731  exposure=ccdExposure,
1732  ossThumb=ossThumb,
1733  flattenedThumb=flattenedThumb,
1734 
1735  preInterpExposure=preInterpExp,
1736  outputExposure=ccdExposure,
1737  outputOssThumbnail=ossThumb,
1738  outputFlattenedThumbnail=flattenedThumb,
1739  )
1740 
1741  @pipeBase.timeMethod
1742  def runDataRef(self, sensorRef):
1743  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1744 
1745  This method contains the `CmdLineTask` interface to the ISR
1746  processing. All IO is handled here, freeing the `run()` method
1747  to manage only pixel-level calculations. The steps performed
1748  are:
1749  - Read in necessary detrending/isr/calibration data.
1750  - Process raw exposure in `run()`.
1751  - Persist the ISR-corrected exposure as "postISRCCD" if
1752  config.doWrite=True.
1753 
1754  Parameters
1755  ----------
1756  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1757  DataRef of the detector data to be processed
1758 
1759  Returns
1760  -------
1761  result : `lsst.pipe.base.Struct`
1762  Result struct with component:
1763  - ``exposure`` : `afw.image.Exposure`
1764  The fully ISR corrected exposure.
1765 
1766  Raises
1767  ------
1768  RuntimeError
1769  Raised if a configuration option is set to True, but the
1770  required calibration data does not exist.
1771 
1772  """
1773  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1774 
1775  ccdExposure = sensorRef.get(self.config.datasetType)
1776 
1777  camera = sensorRef.get("camera")
1778  isrData = self.readIsrDatareadIsrData(sensorRef, ccdExposure)
1779 
1780  result = self.runrun(ccdExposure, camera=camera, **isrData.getDict())
1781 
1782  if self.config.doWrite:
1783  sensorRef.put(result.exposure, "postISRCCD")
1784  if result.preInterpExposure is not None:
1785  sensorRef.put(result.preInterpExposure, "postISRCCD_uninterpolated")
1786  if result.ossThumb is not None:
1787  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1788  if result.flattenedThumb is not None:
1789  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1790 
1791  return result
1792 
1793  def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1794  """Retrieve a calibration dataset for removing instrument signature.
1795 
1796  Parameters
1797  ----------
1798 
1799  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1800  DataRef of the detector data to find calibration datasets
1801  for.
1802  datasetType : `str`
1803  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1804  dateObs : `str`, optional
1805  Date of the observation. Used to correct butler failures
1806  when using fallback filters.
1807  immediate : `Bool`
1808  If True, disable butler proxies to enable error handling
1809  within this routine.
1810 
1811  Returns
1812  -------
1813  exposure : `lsst.afw.image.Exposure`
1814  Requested calibration frame.
1815 
1816  Raises
1817  ------
1818  RuntimeError
1819  Raised if no matching calibration frame can be found.
1820  """
1821  try:
1822  exp = dataRef.get(datasetType, immediate=immediate)
1823  except Exception as exc1:
1824  if not self.config.fallbackFilterName:
1825  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1826  try:
1827  if self.config.useFallbackDate and dateObs:
1828  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1829  dateObs=dateObs, immediate=immediate)
1830  else:
1831  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1832  except Exception as exc2:
1833  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1834  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1835  self.log.warning("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1836 
1837  if self.config.doAssembleIsrExposures:
1838  exp = self.assembleCcd.assembleCcd(exp)
1839  return exp
1840 
1841  def ensureExposure(self, inputExp, camera, detectorNum):
1842  """Ensure that the data returned by Butler is a fully constructed exposure.
1843 
1844  ISR requires exposure-level image data for historical reasons, so if we did
1845  not recieve that from Butler, construct it from what we have, modifying the
1846  input in place.
1847 
1848  Parameters
1849  ----------
1850  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1851  `lsst.afw.image.ImageF`
1852  The input data structure obtained from Butler.
1853  camera : `lsst.afw.cameraGeom.camera`
1854  The camera associated with the image. Used to find the appropriate
1855  detector.
1856  detectorNum : `int`
1857  The detector this exposure should match.
1858 
1859  Returns
1860  -------
1861  inputExp : `lsst.afw.image.Exposure`
1862  The re-constructed exposure, with appropriate detector parameters.
1863 
1864  Raises
1865  ------
1866  TypeError
1867  Raised if the input data cannot be used to construct an exposure.
1868  """
1869  if isinstance(inputExp, afwImage.DecoratedImageU):
1870  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1871  elif isinstance(inputExp, afwImage.ImageF):
1872  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1873  elif isinstance(inputExp, afwImage.MaskedImageF):
1874  inputExp = afwImage.makeExposure(inputExp)
1875  elif isinstance(inputExp, afwImage.Exposure):
1876  pass
1877  elif inputExp is None:
1878  # Assume this will be caught by the setup if it is a problem.
1879  return inputExp
1880  else:
1881  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1882  (type(inputExp), ))
1883 
1884  if inputExp.getDetector() is None:
1885  inputExp.setDetector(camera[detectorNum])
1886 
1887  return inputExp
1888 
1889  def convertIntToFloat(self, exposure):
1890  """Convert exposure image from uint16 to float.
1891 
1892  If the exposure does not need to be converted, the input is
1893  immediately returned. For exposures that are converted to use
1894  floating point pixels, the variance is set to unity and the
1895  mask to zero.
1896 
1897  Parameters
1898  ----------
1899  exposure : `lsst.afw.image.Exposure`
1900  The raw exposure to be converted.
1901 
1902  Returns
1903  -------
1904  newexposure : `lsst.afw.image.Exposure`
1905  The input ``exposure``, converted to floating point pixels.
1906 
1907  Raises
1908  ------
1909  RuntimeError
1910  Raised if the exposure type cannot be converted to float.
1911 
1912  """
1913  if isinstance(exposure, afwImage.ExposureF):
1914  # Nothing to be done
1915  self.log.debug("Exposure already of type float.")
1916  return exposure
1917  if not hasattr(exposure, "convertF"):
1918  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1919 
1920  newexposure = exposure.convertF()
1921  newexposure.variance[:] = 1
1922  newexposure.mask[:] = 0x0
1923 
1924  return newexposure
1925 
1926  def maskAmplifier(self, ccdExposure, amp, defects):
1927  """Identify bad amplifiers, saturated and suspect pixels.
1928 
1929  Parameters
1930  ----------
1931  ccdExposure : `lsst.afw.image.Exposure`
1932  Input exposure to be masked.
1933  amp : `lsst.afw.table.AmpInfoCatalog`
1934  Catalog of parameters defining the amplifier on this
1935  exposure to mask.
1936  defects : `lsst.ip.isr.Defects`
1937  List of defects. Used to determine if the entire
1938  amplifier is bad.
1939 
1940  Returns
1941  -------
1942  badAmp : `Bool`
1943  If this is true, the entire amplifier area is covered by
1944  defects and unusable.
1945 
1946  """
1947  maskedImage = ccdExposure.getMaskedImage()
1948 
1949  badAmp = False
1950 
1951  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1952  # comparison with current defects definition.
1953  if defects is not None:
1954  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1955 
1956  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1957  # association with pixels in current ccdExposure).
1958  if badAmp:
1959  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1960  afwImage.PARENT)
1961  maskView = dataView.getMask()
1962  maskView |= maskView.getPlaneBitMask("BAD")
1963  del maskView
1964  return badAmp
1965 
1966  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1967  # Saturation and suspect pixels can be masked now, though.
1968  limits = dict()
1969  if self.config.doSaturation and not badAmp:
1970  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1971  if self.config.doSuspect and not badAmp:
1972  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1973  if math.isfinite(self.config.saturation):
1974  limits.update({self.config.saturatedMaskName: self.config.saturation})
1975 
1976  for maskName, maskThreshold in limits.items():
1977  if not math.isnan(maskThreshold):
1978  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1979  isrFunctions.makeThresholdMask(
1980  maskedImage=dataView,
1981  threshold=maskThreshold,
1982  growFootprints=0,
1983  maskName=maskName
1984  )
1985 
1986  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1987  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1988  afwImage.PARENT)
1989  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1990  self.config.suspectMaskName])
1991  if numpy.all(maskView.getArray() & maskVal > 0):
1992  badAmp = True
1993  maskView |= maskView.getPlaneBitMask("BAD")
1994 
1995  return badAmp
1996 
1997  def overscanCorrection(self, ccdExposure, amp):
1998  """Apply overscan correction in place.
1999 
2000  This method does initial pixel rejection of the overscan
2001  region. The overscan can also be optionally segmented to
2002  allow for discontinuous overscan responses to be fit
2003  separately. The actual overscan subtraction is performed by
2004  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
2005  which is called here after the amplifier is preprocessed.
2006 
2007  Parameters
2008  ----------
2009  ccdExposure : `lsst.afw.image.Exposure`
2010  Exposure to have overscan correction performed.
2011  amp : `lsst.afw.cameraGeom.Amplifer`
2012  The amplifier to consider while correcting the overscan.
2013 
2014  Returns
2015  -------
2016  overscanResults : `lsst.pipe.base.Struct`
2017  Result struct with components:
2018  - ``imageFit`` : scalar or `lsst.afw.image.Image`
2019  Value or fit subtracted from the amplifier image data.
2020  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
2021  Value or fit subtracted from the overscan image data.
2022  - ``overscanImage`` : `lsst.afw.image.Image`
2023  Image of the overscan region with the overscan
2024  correction applied. This quantity is used to estimate
2025  the amplifier read noise empirically.
2026 
2027  Raises
2028  ------
2029  RuntimeError
2030  Raised if the ``amp`` does not contain raw pixel information.
2031 
2032  See Also
2033  --------
2034  lsst.ip.isr.isrFunctions.overscanCorrection
2035  """
2036  if amp.getRawHorizontalOverscanBBox().isEmpty():
2037  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
2038  return None
2039 
2040  statControl = afwMath.StatisticsControl()
2041  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2042 
2043  # Determine the bounding boxes
2044  dataBBox = amp.getRawDataBBox()
2045  oscanBBox = amp.getRawHorizontalOverscanBBox()
2046  dx0 = 0
2047  dx1 = 0
2048 
2049  prescanBBox = amp.getRawPrescanBBox()
2050  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
2051  dx0 += self.config.overscanNumLeadingColumnsToSkip
2052  dx1 -= self.config.overscanNumTrailingColumnsToSkip
2053  else:
2054  dx0 += self.config.overscanNumTrailingColumnsToSkip
2055  dx1 -= self.config.overscanNumLeadingColumnsToSkip
2056 
2057  # Determine if we need to work on subregions of the amplifier and overscan.
2058  imageBBoxes = []
2059  overscanBBoxes = []
2060 
2061  if ((self.config.overscanBiasJump
2062  and self.config.overscanBiasJumpLocation)
2063  and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2064  and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
2065  self.config.overscanBiasJumpDevices)):
2066  if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
2067  yLower = self.config.overscanBiasJumpLocation
2068  yUpper = dataBBox.getHeight() - yLower
2069  else:
2070  yUpper = self.config.overscanBiasJumpLocation
2071  yLower = dataBBox.getHeight() - yUpper
2072 
2073  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2074  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
2075  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2076  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2077  yLower)))
2078 
2079  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
2080  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
2081  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
2082  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2083  yUpper)))
2084  else:
2085  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2086  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
2087  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2088  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2089  oscanBBox.getHeight())))
2090 
2091  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
2092  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
2093  ampImage = ccdExposure.maskedImage[imageBBox]
2094  overscanImage = ccdExposure.maskedImage[overscanBBox]
2095 
2096  overscanArray = overscanImage.image.array
2097  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2098  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2099  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
2100 
2101  statControl = afwMath.StatisticsControl()
2102  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2103 
2104  overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp)
2105 
2106  # Measure average overscan levels and record them in the metadata.
2107  levelStat = afwMath.MEDIAN
2108  sigmaStat = afwMath.STDEVCLIP
2109 
2110  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2111  self.config.qa.flatness.nIter)
2112  metadata = ccdExposure.getMetadata()
2113  ampNum = amp.getName()
2114  # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
2115  if isinstance(overscanResults.overscanFit, float):
2116  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2117  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2118  else:
2119  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2120  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2121  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2122 
2123  return overscanResults
2124 
2125  def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None):
2126  """Set the variance plane using the gain and read noise
2127 
2128  The read noise is calculated from the ``overscanImage`` if the
2129  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2130  the value from the amplifier data is used.
2131 
2132  Parameters
2133  ----------
2134  ampExposure : `lsst.afw.image.Exposure`
2135  Exposure to process.
2136  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2137  Amplifier detector data.
2138  overscanImage : `lsst.afw.image.MaskedImage`, optional.
2139  Image of overscan, required only for empirical read noise.
2140  ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
2141  PTC dataset containing the gains and read noise.
2142 
2143 
2144  Raises
2145  ------
2146  RuntimeError
2147  Raised if either ``usePtcGains`` of ``usePtcReadNoise``
2148  are ``True``, but ptcDataset is not provided.
2149 
2150  Raised if ```doEmpiricalReadNoise`` is ``True`` but
2151  ``overscanImage`` is ``None``.
2152 
2153  See also
2154  --------
2155  lsst.ip.isr.isrFunctions.updateVariance
2156  """
2157  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2158  if self.config.usePtcGains:
2159  if ptcDataset is None:
2160  raise RuntimeError("No ptcDataset provided to use PTC gains.")
2161  else:
2162  gain = ptcDataset.gain[amp.getName()]
2163  self.log.info("Using gain from Photon Transfer Curve.")
2164  else:
2165  gain = amp.getGain()
2166 
2167  if math.isnan(gain):
2168  gain = 1.0
2169  self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2170  elif gain <= 0:
2171  patchedGain = 1.0
2172  self.log.warning("Gain for amp %s == %g <= 0; setting to %f.",
2173  amp.getName(), gain, patchedGain)
2174  gain = patchedGain
2175 
2176  if self.config.doEmpiricalReadNoise and overscanImage is None:
2177  raise RuntimeError("Overscan is none for EmpiricalReadNoise.")
2178 
2179  if self.config.doEmpiricalReadNoise and overscanImage is not None:
2180  stats = afwMath.StatisticsControl()
2181  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2182  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2183  self.log.info("Calculated empirical read noise for amp %s: %f.",
2184  amp.getName(), readNoise)
2185  elif self.config.usePtcReadNoise:
2186  if ptcDataset is None:
2187  raise RuntimeError("No ptcDataset provided to use PTC readnoise.")
2188  else:
2189  readNoise = ptcDataset.noise[amp.getName()]
2190  self.log.info("Using read noise from Photon Transfer Curve.")
2191  else:
2192  readNoise = amp.getReadNoise()
2193 
2194  isrFunctions.updateVariance(
2195  maskedImage=ampExposure.getMaskedImage(),
2196  gain=gain,
2197  readNoise=readNoise,
2198  )
2199 
2200  def maskNegativeVariance(self, exposure):
2201  """Identify and mask pixels with negative variance values.
2202 
2203  Parameters
2204  ----------
2205  exposure : `lsst.afw.image.Exposure`
2206  Exposure to process.
2207 
2208  See Also
2209  --------
2210  lsst.ip.isr.isrFunctions.updateVariance
2211  """
2212  maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
2213  bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
2214  exposure.mask.array[bad] |= maskPlane
2215 
2216  def darkCorrection(self, exposure, darkExposure, invert=False):
2217  """Apply dark correction in place.
2218 
2219  Parameters
2220  ----------
2221  exposure : `lsst.afw.image.Exposure`
2222  Exposure to process.
2223  darkExposure : `lsst.afw.image.Exposure`
2224  Dark exposure of the same size as ``exposure``.
2225  invert : `Bool`, optional
2226  If True, re-add the dark to an already corrected image.
2227 
2228  Raises
2229  ------
2230  RuntimeError
2231  Raised if either ``exposure`` or ``darkExposure`` do not
2232  have their dark time defined.
2233 
2234  See Also
2235  --------
2236  lsst.ip.isr.isrFunctions.darkCorrection
2237  """
2238  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2239  if math.isnan(expScale):
2240  raise RuntimeError("Exposure darktime is NAN.")
2241  if darkExposure.getInfo().getVisitInfo() is not None \
2242  and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2243  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2244  else:
2245  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
2246  # so getDarkTime() does not exist.
2247  self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2248  darkScale = 1.0
2249 
2250  isrFunctions.darkCorrection(
2251  maskedImage=exposure.getMaskedImage(),
2252  darkMaskedImage=darkExposure.getMaskedImage(),
2253  expScale=expScale,
2254  darkScale=darkScale,
2255  invert=invert,
2256  trimToFit=self.config.doTrimToMatchCalib
2257  )
2258 
2259  def doLinearize(self, detector):
2260  """Check if linearization is needed for the detector cameraGeom.
2261 
2262  Checks config.doLinearize and the linearity type of the first
2263  amplifier.
2264 
2265  Parameters
2266  ----------
2267  detector : `lsst.afw.cameraGeom.Detector`
2268  Detector to get linearity type from.
2269 
2270  Returns
2271  -------
2272  doLinearize : `Bool`
2273  If True, linearization should be performed.
2274  """
2275  return self.config.doLinearize and \
2276  detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2277 
2278  def flatCorrection(self, exposure, flatExposure, invert=False):
2279  """Apply flat correction in place.
2280 
2281  Parameters
2282  ----------
2283  exposure : `lsst.afw.image.Exposure`
2284  Exposure to process.
2285  flatExposure : `lsst.afw.image.Exposure`
2286  Flat exposure of the same size as ``exposure``.
2287  invert : `Bool`, optional
2288  If True, unflatten an already flattened image.
2289 
2290  See Also
2291  --------
2292  lsst.ip.isr.isrFunctions.flatCorrection
2293  """
2294  isrFunctions.flatCorrection(
2295  maskedImage=exposure.getMaskedImage(),
2296  flatMaskedImage=flatExposure.getMaskedImage(),
2297  scalingType=self.config.flatScalingType,
2298  userScale=self.config.flatUserScale,
2299  invert=invert,
2300  trimToFit=self.config.doTrimToMatchCalib
2301  )
2302 
2303  def saturationDetection(self, exposure, amp):
2304  """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2305 
2306  Parameters
2307  ----------
2308  exposure : `lsst.afw.image.Exposure`
2309  Exposure to process. Only the amplifier DataSec is processed.
2310  amp : `lsst.afw.table.AmpInfoCatalog`
2311  Amplifier detector data.
2312 
2313  See Also
2314  --------
2315  lsst.ip.isr.isrFunctions.makeThresholdMask
2316  """
2317  if not math.isnan(amp.getSaturation()):
2318  maskedImage = exposure.getMaskedImage()
2319  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2320  isrFunctions.makeThresholdMask(
2321  maskedImage=dataView,
2322  threshold=amp.getSaturation(),
2323  growFootprints=0,
2324  maskName=self.config.saturatedMaskName,
2325  )
2326 
2327  def saturationInterpolation(self, exposure):
2328  """Interpolate over saturated pixels, in place.
2329 
2330  This method should be called after `saturationDetection`, to
2331  ensure that the saturated pixels have been identified in the
2332  SAT mask. It should also be called after `assembleCcd`, since
2333  saturated regions may cross amplifier boundaries.
2334 
2335  Parameters
2336  ----------
2337  exposure : `lsst.afw.image.Exposure`
2338  Exposure to process.
2339 
2340  See Also
2341  --------
2342  lsst.ip.isr.isrTask.saturationDetection
2343  lsst.ip.isr.isrFunctions.interpolateFromMask
2344  """
2345  isrFunctions.interpolateFromMask(
2346  maskedImage=exposure.getMaskedImage(),
2347  fwhm=self.config.fwhm,
2348  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2349  maskNameList=list(self.config.saturatedMaskName),
2350  )
2351 
2352  def suspectDetection(self, exposure, amp):
2353  """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2354 
2355  Parameters
2356  ----------
2357  exposure : `lsst.afw.image.Exposure`
2358  Exposure to process. Only the amplifier DataSec is processed.
2359  amp : `lsst.afw.table.AmpInfoCatalog`
2360  Amplifier detector data.
2361 
2362  See Also
2363  --------
2364  lsst.ip.isr.isrFunctions.makeThresholdMask
2365 
2366  Notes
2367  -----
2368  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2369  This is intended to indicate pixels that may be affected by unknown systematics;
2370  for example if non-linearity corrections above a certain level are unstable
2371  then that would be a useful value for suspectLevel. A value of `nan` indicates
2372  that no such level exists and no pixels are to be masked as suspicious.
2373  """
2374  suspectLevel = amp.getSuspectLevel()
2375  if math.isnan(suspectLevel):
2376  return
2377 
2378  maskedImage = exposure.getMaskedImage()
2379  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2380  isrFunctions.makeThresholdMask(
2381  maskedImage=dataView,
2382  threshold=suspectLevel,
2383  growFootprints=0,
2384  maskName=self.config.suspectMaskName,
2385  )
2386 
2387  def maskDefect(self, exposure, defectBaseList):
2388  """Mask defects using mask plane "BAD", in place.
2389 
2390  Parameters
2391  ----------
2392  exposure : `lsst.afw.image.Exposure`
2393  Exposure to process.
2394  defectBaseList : `lsst.ip.isr.Defects` or `list` of
2395  `lsst.afw.image.DefectBase`.
2396  List of defects to mask.
2397 
2398  Notes
2399  -----
2400  Call this after CCD assembly, since defects may cross amplifier boundaries.
2401  """
2402  maskedImage = exposure.getMaskedImage()
2403  if not isinstance(defectBaseList, Defects):
2404  # Promotes DefectBase to Defect
2405  defectList = Defects(defectBaseList)
2406  else:
2407  defectList = defectBaseList
2408  defectList.maskPixels(maskedImage, maskName="BAD")
2409 
2410  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2411  """Mask edge pixels with applicable mask plane.
2412 
2413  Parameters
2414  ----------
2415  exposure : `lsst.afw.image.Exposure`
2416  Exposure to process.
2417  numEdgePixels : `int`, optional
2418  Number of edge pixels to mask.
2419  maskPlane : `str`, optional
2420  Mask plane name to use.
2421  level : `str`, optional
2422  Level at which to mask edges.
2423  """
2424  maskedImage = exposure.getMaskedImage()
2425  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2426 
2427  if numEdgePixels > 0:
2428  if level == 'DETECTOR':
2429  boxes = [maskedImage.getBBox()]
2430  elif level == 'AMP':
2431  boxes = [amp.getBBox() for amp in exposure.getDetector()]
2432 
2433  for box in boxes:
2434  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2435  subImage = maskedImage[box]
2436  box.grow(-numEdgePixels)
2437  # Mask pixels outside box
2438  SourceDetectionTask.setEdgeBits(
2439  subImage,
2440  box,
2441  maskBitMask)
2442 
2443  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2444  """Mask and interpolate defects using mask plane "BAD", in place.
2445 
2446  Parameters
2447  ----------
2448  exposure : `lsst.afw.image.Exposure`
2449  Exposure to process.
2450  defectBaseList : `lsst.ip.isr.Defects` or `list` of
2451  `lsst.afw.image.DefectBase`.
2452  List of defects to mask and interpolate.
2453 
2454  See Also
2455  --------
2456  lsst.ip.isr.isrTask.maskDefect
2457  """
2458  self.maskDefectmaskDefect(exposure, defectBaseList)
2459  self.maskEdgesmaskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2460  maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2461  isrFunctions.interpolateFromMask(
2462  maskedImage=exposure.getMaskedImage(),
2463  fwhm=self.config.fwhm,
2464  growSaturatedFootprints=0,
2465  maskNameList=["BAD"],
2466  )
2467 
2468  def maskNan(self, exposure):
2469  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2470 
2471  Parameters
2472  ----------
2473  exposure : `lsst.afw.image.Exposure`
2474  Exposure to process.
2475 
2476  Notes
2477  -----
2478  We mask over all non-finite values (NaN, inf), including those
2479  that are masked with other bits (because those may or may not be
2480  interpolated over later, and we want to remove all NaN/infs).
2481  Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2482  preserve the historical name.
2483  """
2484  maskedImage = exposure.getMaskedImage()
2485 
2486  # Find and mask NaNs
2487  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2488  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2489  numNans = maskNans(maskedImage, maskVal)
2490  self.metadata.set("NUMNANS", numNans)
2491  if numNans > 0:
2492  self.log.warning("There were %d unmasked NaNs.", numNans)
2493 
2494  def maskAndInterpolateNan(self, exposure):
2495  """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2496  in place.
2497 
2498  Parameters
2499  ----------
2500  exposure : `lsst.afw.image.Exposure`
2501  Exposure to process.
2502 
2503  See Also
2504  --------
2505  lsst.ip.isr.isrTask.maskNan
2506  """
2507  self.maskNanmaskNan(exposure)
2508  isrFunctions.interpolateFromMask(
2509  maskedImage=exposure.getMaskedImage(),
2510  fwhm=self.config.fwhm,
2511  growSaturatedFootprints=0,
2512  maskNameList=["UNMASKEDNAN"],
2513  )
2514 
2515  def measureBackground(self, exposure, IsrQaConfig=None):
2516  """Measure the image background in subgrids, for quality control purposes.
2517 
2518  Parameters
2519  ----------
2520  exposure : `lsst.afw.image.Exposure`
2521  Exposure to process.
2522  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2523  Configuration object containing parameters on which background
2524  statistics and subgrids to use.
2525  """
2526  if IsrQaConfig is not None:
2527  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2528  IsrQaConfig.flatness.nIter)
2529  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2530  statsControl.setAndMask(maskVal)
2531  maskedImage = exposure.getMaskedImage()
2532  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2533  skyLevel = stats.getValue(afwMath.MEDIAN)
2534  skySigma = stats.getValue(afwMath.STDEVCLIP)
2535  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2536  metadata = exposure.getMetadata()
2537  metadata.set('SKYLEVEL', skyLevel)
2538  metadata.set('SKYSIGMA', skySigma)
2539 
2540  # calcluating flatlevel over the subgrids
2541  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2542  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2543  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2544  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2545  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2546  skyLevels = numpy.zeros((nX, nY))
2547 
2548  for j in range(nY):
2549  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2550  for i in range(nX):
2551  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2552 
2553  xLLC = xc - meshXHalf
2554  yLLC = yc - meshYHalf
2555  xURC = xc + meshXHalf - 1
2556  yURC = yc + meshYHalf - 1
2557 
2558  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2559  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2560 
2561  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2562 
2563  good = numpy.where(numpy.isfinite(skyLevels))
2564  skyMedian = numpy.median(skyLevels[good])
2565  flatness = (skyLevels[good] - skyMedian) / skyMedian
2566  flatness_rms = numpy.std(flatness)
2567  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2568 
2569  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2570  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2571  nX, nY, flatness_pp, flatness_rms)
2572 
2573  metadata.set('FLATNESS_PP', float(flatness_pp))
2574  metadata.set('FLATNESS_RMS', float(flatness_rms))
2575  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2576  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2577  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2578 
2579  def roughZeroPoint(self, exposure):
2580  """Set an approximate magnitude zero point for the exposure.
2581 
2582  Parameters
2583  ----------
2584  exposure : `lsst.afw.image.Exposure`
2585  Exposure to process.
2586  """
2587  filterLabel = exposure.getFilterLabel()
2588  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2589 
2590  if physicalFilter in self.config.fluxMag0T1:
2591  fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2592  else:
2593  self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter)
2594  fluxMag0 = self.config.defaultFluxMag0T1
2595 
2596  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2597  if not expTime > 0: # handle NaN as well as <= 0
2598  self.log.warning("Non-positive exposure time; skipping rough zero point.")
2599  return
2600 
2601  self.log.info("Setting rough magnitude zero point for filter %s: %f",
2602  physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2603  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2604 
2605  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2606  """Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2607 
2608  Parameters
2609  ----------
2610  ccdExposure : `lsst.afw.image.Exposure`
2611  Exposure to process.
2612  fpPolygon : `lsst.afw.geom.Polygon`
2613  Polygon in focal plane coordinates.
2614  """
2615  # Get ccd corners in focal plane coordinates
2616  ccd = ccdExposure.getDetector()
2617  fpCorners = ccd.getCorners(FOCAL_PLANE)
2618  ccdPolygon = Polygon(fpCorners)
2619 
2620  # Get intersection of ccd corners with fpPolygon
2621  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2622 
2623  # Transform back to pixel positions and build new polygon
2624  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2625  validPolygon = Polygon(ccdPoints)
2626  ccdExposure.getInfo().setValidPolygon(validPolygon)
2627 
2628  @contextmanager
2629  def flatContext(self, exp, flat, dark=None):
2630  """Context manager that applies and removes flats and darks,
2631  if the task is configured to apply them.
2632 
2633  Parameters
2634  ----------
2635  exp : `lsst.afw.image.Exposure`
2636  Exposure to process.
2637  flat : `lsst.afw.image.Exposure`
2638  Flat exposure the same size as ``exp``.
2639  dark : `lsst.afw.image.Exposure`, optional
2640  Dark exposure the same size as ``exp``.
2641 
2642  Yields
2643  ------
2644  exp : `lsst.afw.image.Exposure`
2645  The flat and dark corrected exposure.
2646  """
2647  if self.config.doDark and dark is not None:
2648  self.darkCorrectiondarkCorrection(exp, dark)
2649  if self.config.doFlat:
2650  self.flatCorrectionflatCorrection(exp, flat)
2651  try:
2652  yield exp
2653  finally:
2654  if self.config.doFlat:
2655  self.flatCorrectionflatCorrection(exp, flat, invert=True)
2656  if self.config.doDark and dark is not None:
2657  self.darkCorrectiondarkCorrection(exp, dark, invert=True)
2658 
2659  def debugView(self, exposure, stepname):
2660  """Utility function to examine ISR exposure at different stages.
2661 
2662  Parameters
2663  ----------
2664  exposure : `lsst.afw.image.Exposure`
2665  Exposure to view.
2666  stepname : `str`
2667  State of processing to view.
2668  """
2669  frame = getDebugFrame(self._display, stepname)
2670  if frame:
2671  display = getDisplay(frame)
2672  display.scale('asinh', 'zscale')
2673  display.mtv(exposure)
2674  prompt = "Press Enter to continue [c]... "
2675  while True:
2676  ans = input(prompt).lower()
2677  if ans in ("", "c",):
2678  break
2679 
2680 
2681 class FakeAmp(object):
2682  """A Detector-like object that supports returning gain and saturation level
2683 
2684  This is used when the input exposure does not have a detector.
2685 
2686  Parameters
2687  ----------
2688  exposure : `lsst.afw.image.Exposure`
2689  Exposure to generate a fake amplifier for.
2690  config : `lsst.ip.isr.isrTaskConfig`
2691  Configuration to apply to the fake amplifier.
2692  """
2693 
2694  def __init__(self, exposure, config):
2695  self._bbox_bbox = exposure.getBBox(afwImage.LOCAL)
2696  self._RawHorizontalOverscanBBox_RawHorizontalOverscanBBox = lsst.geom.Box2I()
2697  self._gain_gain = config.gain
2698  self._readNoise_readNoise = config.readNoise
2699  self._saturation_saturation = config.saturation
2700 
2701  def getBBox(self):
2702  return self._bbox_bbox
2703 
2704  def getRawBBox(self):
2705  return self._bbox_bbox
2706 
2708  return self._RawHorizontalOverscanBBox_RawHorizontalOverscanBBox
2709 
2710  def getGain(self):
2711  return self._gain_gain
2712 
2713  def getReadNoise(self):
2714  return self._readNoise_readNoise
2715 
2716  def getSaturation(self):
2717  return self._saturation_saturation
2718 
2719  def getSuspectLevel(self):
2720  return float("NaN")
2721 
2722 
2723 class RunIsrConfig(pexConfig.Config):
2724  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2725 
2726 
2727 class RunIsrTask(pipeBase.CmdLineTask):
2728  """Task to wrap the default IsrTask to allow it to be retargeted.
2729 
2730  The standard IsrTask can be called directly from a command line
2731  program, but doing so removes the ability of the task to be
2732  retargeted. As most cameras override some set of the IsrTask
2733  methods, this would remove those data-specific methods in the
2734  output post-ISR images. This wrapping class fixes the issue,
2735  allowing identical post-ISR images to be generated by both the
2736  processCcd and isrTask code.
2737  """
2738  ConfigClass = RunIsrConfig
2739  _DefaultName = "runIsr"
2740 
2741  def __init__(self, *args, **kwargs):
2742  super().__init__(*args, **kwargs)
2743  self.makeSubtask("isr")
2744 
2745  def runDataRef(self, dataRef):
2746  """
2747  Parameters
2748  ----------
2749  dataRef : `lsst.daf.persistence.ButlerDataRef`
2750  data reference of the detector data to be processed
2751 
2752  Returns
2753  -------
2754  result : `pipeBase.Struct`
2755  Result struct with component:
2756 
2757  - exposure : `lsst.afw.image.Exposure`
2758  Post-ISR processed exposure.
2759  """
2760  return self.isr.runDataRef(dataRef)
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2707
def __init__(self, exposure, config)
Definition: isrTask.py:2694
def __init__(self, *config=None)
Definition: isrTask.py:272
def flatCorrection(self, exposure, flatExposure, invert=False)
Definition: isrTask.py:2278
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2494
def saturationInterpolation(self, exposure)
Definition: isrTask.py:2327
def runDataRef(self, sensorRef)
Definition: isrTask.py:1742
def maskNan(self, exposure)
Definition: isrTask.py:2468
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1926
def debugView(self, exposure, stepname)
Definition: isrTask.py:2659
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Definition: isrTask.py:1793
def maskNegativeVariance(self, exposure)
Definition: isrTask.py:2200
def saturationDetection(self, exposure, amp)
Definition: isrTask.py:2303
def maskDefect(self, exposure, defectBaseList)
Definition: isrTask.py:2387
def __init__(self, **kwargs)
Definition: isrTask.py:960
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:970
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
Definition: isrTask.py:2410
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1997
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2515
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2579
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2443
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Definition: isrTask.py:2605
def readIsrData(self, dataRef, rawExposure)
Definition: isrTask.py:1068
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1841
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Definition: isrTask.py:1250
def doLinearize(self, detector)
Definition: isrTask.py:2259
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2629
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1889
def suspectDetection(self, exposure, amp)
Definition: isrTask.py:2352
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
Definition: isrTask.py:2125
def darkCorrection(self, exposure, darkExposure, invert=False)
Definition: isrTask.py:2216
def __init__(self, *args, **kwargs)
Definition: isrTask.py:2741
def runDataRef(self, dataRef)
Definition: isrTask.py:2745
def checkFilter(exposure, filterList, log)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
Definition: isrTask.py:62
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35