lsst.ip.isr  22.0.0-11-g57c5a76+f06137c6a5
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
30 import lsst.pipe.base.connectionTypes as cT
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36  ReadoutCorner)
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 
43 from . import isrFunctions
44 from . import isrQa
45 from . import linearize
46 from .defects import Defects
47 
48 from .assembleCcdTask import AssembleCcdTask
49 from .crosstalk import CrosstalkTask, CrosstalkCalib
50 from .fringe import FringeTask
51 from .isr import maskNans
52 from .masking import MaskingTask
53 from .overscan import OverscanCorrectionTask
54 from .straylight import StrayLightTask
55 from .vignette import VignetteTask
56 from lsst.daf.butler import DimensionGraph
57 
58 
59 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
60 
61 
62 def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections):
63  """Lookup function to identify crosstalkSource entries.
64 
65  This should return an empty list under most circumstances. Only
66  when inter-chip crosstalk has been identified should this be
67  populated.
68 
69  This will be unused until DM-25348 resolves the quantum graph
70  generation issue.
71 
72  Parameters
73  ----------
74  datasetType : `str`
75  Dataset to lookup.
76  registry : `lsst.daf.butler.Registry`
77  Butler registry to query.
78  quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79  Data id to transform to identify crosstalkSources. The
80  ``detector`` entry will be stripped.
81  collections : `lsst.daf.butler.CollectionSearch`
82  Collections to search through.
83 
84  Returns
85  -------
86  results : `list` [`lsst.daf.butler.DatasetRef`]
87  List of datasets that match the query that will be used as
88  crosstalkSources.
89  """
90  newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"]))
91  results = list(registry.queryDatasets(datasetType,
92  collections=collections,
93  dataId=newDataId,
94  findFirst=True,
95  ).expanded())
96  return results
97 
98 
99 class IsrTaskConnections(pipeBase.PipelineTaskConnections,
100  dimensions={"instrument", "exposure", "detector"},
101  defaultTemplates={}):
102  ccdExposure = cT.Input(
103  name="raw",
104  doc="Input exposure to process.",
105  storageClass="Exposure",
106  dimensions=["instrument", "exposure", "detector"],
107  )
108  camera = cT.PrerequisiteInput(
109  name="camera",
110  storageClass="Camera",
111  doc="Input camera to construct complete exposures.",
112  dimensions=["instrument"],
113  isCalibration=True,
114  )
115 
116  crosstalk = cT.PrerequisiteInput(
117  name="crosstalk",
118  doc="Input crosstalk object",
119  storageClass="CrosstalkCalib",
120  dimensions=["instrument", "detector"],
121  isCalibration=True,
122  minimum=0, # can fall back to cameraGeom
123  )
124  # TODO: DM-25348. This does not work yet to correctly load
125  # possible crosstalk sources.
126  crosstalkSources = cT.PrerequisiteInput(
127  name="isrOverscanCorrected",
128  doc="Overscan corrected input images.",
129  storageClass="Exposure",
130  dimensions=["instrument", "exposure", "detector"],
131  deferLoad=True,
132  multiple=True,
133  lookupFunction=crosstalkSourceLookup,
134  minimum=0, # not needed for all instruments, no config to control this
135  )
136  bias = cT.PrerequisiteInput(
137  name="bias",
138  doc="Input bias calibration.",
139  storageClass="ExposureF",
140  dimensions=["instrument", "detector"],
141  isCalibration=True,
142  )
143  dark = cT.PrerequisiteInput(
144  name='dark',
145  doc="Input dark calibration.",
146  storageClass="ExposureF",
147  dimensions=["instrument", "detector"],
148  isCalibration=True,
149  )
150  flat = cT.PrerequisiteInput(
151  name="flat",
152  doc="Input flat calibration.",
153  storageClass="ExposureF",
154  dimensions=["instrument", "physical_filter", "detector"],
155  isCalibration=True,
156  )
157  ptc = cT.PrerequisiteInput(
158  name="ptc",
159  doc="Input Photon Transfer Curve dataset",
160  storageClass="PhotonTransferCurveDataset",
161  dimensions=["instrument", "detector"],
162  isCalibration=True,
163  )
164  fringes = cT.PrerequisiteInput(
165  name="fringe",
166  doc="Input fringe calibration.",
167  storageClass="ExposureF",
168  dimensions=["instrument", "physical_filter", "detector"],
169  isCalibration=True,
170  minimum=0, # only needed for some bands, even when enabled
171  )
172  strayLightData = cT.PrerequisiteInput(
173  name='yBackground',
174  doc="Input stray light calibration.",
175  storageClass="StrayLightData",
176  dimensions=["instrument", "physical_filter", "detector"],
177  isCalibration=True,
178  minimum=0, # only needed for some bands, even when enabled
179  )
180  bfKernel = cT.PrerequisiteInput(
181  name='bfKernel',
182  doc="Input brighter-fatter kernel.",
183  storageClass="NumpyArray",
184  dimensions=["instrument"],
185  isCalibration=True,
186  minimum=0, # can use either bfKernel or newBFKernel
187  )
188  newBFKernel = cT.PrerequisiteInput(
189  name='brighterFatterKernel',
190  doc="Newer complete kernel + gain solutions.",
191  storageClass="BrighterFatterKernel",
192  dimensions=["instrument", "detector"],
193  isCalibration=True,
194  minimum=0, # can use either bfKernel or newBFKernel
195  )
196  defects = cT.PrerequisiteInput(
197  name='defects',
198  doc="Input defect tables.",
199  storageClass="Defects",
200  dimensions=["instrument", "detector"],
201  isCalibration=True,
202  )
203  linearizer = cT.PrerequisiteInput(
204  name='linearizer',
205  storageClass="Linearizer",
206  doc="Linearity correction calibration.",
207  dimensions=["instrument", "detector"],
208  isCalibration=True,
209  minimum=0, # can fall back to cameraGeom
210  )
211  opticsTransmission = cT.PrerequisiteInput(
212  name="transmission_optics",
213  storageClass="TransmissionCurve",
214  doc="Transmission curve due to the optics.",
215  dimensions=["instrument"],
216  isCalibration=True,
217  )
218  filterTransmission = cT.PrerequisiteInput(
219  name="transmission_filter",
220  storageClass="TransmissionCurve",
221  doc="Transmission curve due to the filter.",
222  dimensions=["instrument", "physical_filter"],
223  isCalibration=True,
224  )
225  sensorTransmission = cT.PrerequisiteInput(
226  name="transmission_sensor",
227  storageClass="TransmissionCurve",
228  doc="Transmission curve due to the sensor.",
229  dimensions=["instrument", "detector"],
230  isCalibration=True,
231  )
232  atmosphereTransmission = cT.PrerequisiteInput(
233  name="transmission_atmosphere",
234  storageClass="TransmissionCurve",
235  doc="Transmission curve due to the atmosphere.",
236  dimensions=["instrument"],
237  isCalibration=True,
238  )
239  illumMaskedImage = cT.PrerequisiteInput(
240  name="illum",
241  doc="Input illumination correction.",
242  storageClass="MaskedImageF",
243  dimensions=["instrument", "physical_filter", "detector"],
244  isCalibration=True,
245  )
246 
247  outputExposure = cT.Output(
248  name='postISRCCD',
249  doc="Output ISR processed exposure.",
250  storageClass="Exposure",
251  dimensions=["instrument", "exposure", "detector"],
252  )
253  preInterpExposure = cT.Output(
254  name='preInterpISRCCD',
255  doc="Output ISR processed exposure, with pixels left uninterpolated.",
256  storageClass="ExposureF",
257  dimensions=["instrument", "exposure", "detector"],
258  )
259  outputOssThumbnail = cT.Output(
260  name="OssThumb",
261  doc="Output Overscan-subtracted thumbnail image.",
262  storageClass="Thumbnail",
263  dimensions=["instrument", "exposure", "detector"],
264  )
265  outputFlattenedThumbnail = cT.Output(
266  name="FlattenedThumb",
267  doc="Output flat-corrected thumbnail image.",
268  storageClass="Thumbnail",
269  dimensions=["instrument", "exposure", "detector"],
270  )
271 
272  def __init__(self, *, config=None):
273  super().__init__(config=config)
274 
275  if config.doBias is not True:
276  self.prerequisiteInputs.discard("bias")
277  if config.doLinearize is not True:
278  self.prerequisiteInputs.discard("linearizer")
279  if config.doCrosstalk is not True:
280  self.inputs.discard("crosstalkSources")
281  self.prerequisiteInputs.discard("crosstalk")
282  if config.doBrighterFatter is not True:
283  self.prerequisiteInputs.discard("bfKernel")
284  self.prerequisiteInputs.discard("newBFKernel")
285  if config.doDefect is not True:
286  self.prerequisiteInputs.discard("defects")
287  if config.doDark is not True:
288  self.prerequisiteInputs.discard("dark")
289  if config.doFlat is not True:
290  self.prerequisiteInputs.discard("flat")
291  if config.doFringe is not True:
292  self.prerequisiteInputs.discard("fringe")
293  if config.doStrayLight is not True:
294  self.prerequisiteInputs.discard("strayLightData")
295  if config.usePtcGains is not True and config.usePtcReadNoise is not True:
296  self.prerequisiteInputs.discard("ptc")
297  if config.doAttachTransmissionCurve is not True:
298  self.prerequisiteInputs.discard("opticsTransmission")
299  self.prerequisiteInputs.discard("filterTransmission")
300  self.prerequisiteInputs.discard("sensorTransmission")
301  self.prerequisiteInputs.discard("atmosphereTransmission")
302  if config.doUseOpticsTransmission is not True:
303  self.prerequisiteInputs.discard("opticsTransmission")
304  if config.doUseFilterTransmission is not True:
305  self.prerequisiteInputs.discard("filterTransmission")
306  if config.doUseSensorTransmission is not True:
307  self.prerequisiteInputs.discard("sensorTransmission")
308  if config.doUseAtmosphereTransmission is not True:
309  self.prerequisiteInputs.discard("atmosphereTransmission")
310  if config.doIlluminationCorrection is not True:
311  self.prerequisiteInputs.discard("illumMaskedImage")
312 
313  if config.doWrite is not True:
314  self.outputs.discard("outputExposure")
315  self.outputs.discard("preInterpExposure")
316  self.outputs.discard("outputFlattenedThumbnail")
317  self.outputs.discard("outputOssThumbnail")
318  if config.doSaveInterpPixels is not True:
319  self.outputs.discard("preInterpExposure")
320  if config.qa.doThumbnailOss is not True:
321  self.outputs.discard("outputOssThumbnail")
322  if config.qa.doThumbnailFlattened is not True:
323  self.outputs.discard("outputFlattenedThumbnail")
324 
325 
326 class IsrTaskConfig(pipeBase.PipelineTaskConfig,
327  pipelineConnections=IsrTaskConnections):
328  """Configuration parameters for IsrTask.
329 
330  Items are grouped in the order in which they are executed by the task.
331  """
332  datasetType = pexConfig.Field(
333  dtype=str,
334  doc="Dataset type for input data; users will typically leave this alone, "
335  "but camera-specific ISR tasks will override it",
336  default="raw",
337  )
338 
339  fallbackFilterName = pexConfig.Field(
340  dtype=str,
341  doc="Fallback default filter name for calibrations.",
342  optional=True
343  )
344  useFallbackDate = pexConfig.Field(
345  dtype=bool,
346  doc="Pass observation date when using fallback filter.",
347  default=False,
348  )
349  expectWcs = pexConfig.Field(
350  dtype=bool,
351  default=True,
352  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
353  )
354  fwhm = pexConfig.Field(
355  dtype=float,
356  doc="FWHM of PSF in arcseconds.",
357  default=1.0,
358  )
359  qa = pexConfig.ConfigField(
360  dtype=isrQa.IsrQaConfig,
361  doc="QA related configuration options.",
362  )
363 
364  # Image conversion configuration
365  doConvertIntToFloat = pexConfig.Field(
366  dtype=bool,
367  doc="Convert integer raw images to floating point values?",
368  default=True,
369  )
370 
371  # Saturated pixel handling.
372  doSaturation = pexConfig.Field(
373  dtype=bool,
374  doc="Mask saturated pixels? NB: this is totally independent of the"
375  " interpolation option - this is ONLY setting the bits in the mask."
376  " To have them interpolated make sure doSaturationInterpolation=True",
377  default=True,
378  )
379  saturatedMaskName = pexConfig.Field(
380  dtype=str,
381  doc="Name of mask plane to use in saturation detection and interpolation",
382  default="SAT",
383  )
384  saturation = pexConfig.Field(
385  dtype=float,
386  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
387  default=float("NaN"),
388  )
389  growSaturationFootprintSize = pexConfig.Field(
390  dtype=int,
391  doc="Number of pixels by which to grow the saturation footprints",
392  default=1,
393  )
394 
395  # Suspect pixel handling.
396  doSuspect = pexConfig.Field(
397  dtype=bool,
398  doc="Mask suspect pixels?",
399  default=False,
400  )
401  suspectMaskName = pexConfig.Field(
402  dtype=str,
403  doc="Name of mask plane to use for suspect pixels",
404  default="SUSPECT",
405  )
406  numEdgeSuspect = pexConfig.Field(
407  dtype=int,
408  doc="Number of edge pixels to be flagged as untrustworthy.",
409  default=0,
410  )
411  edgeMaskLevel = pexConfig.ChoiceField(
412  dtype=str,
413  doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
414  default="DETECTOR",
415  allowed={
416  'DETECTOR': 'Mask only the edges of the full detector.',
417  'AMP': 'Mask edges of each amplifier.',
418  },
419  )
420 
421  # Initial masking options.
422  doSetBadRegions = pexConfig.Field(
423  dtype=bool,
424  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
425  default=True,
426  )
427  badStatistic = pexConfig.ChoiceField(
428  dtype=str,
429  doc="How to estimate the average value for BAD regions.",
430  default='MEANCLIP',
431  allowed={
432  "MEANCLIP": "Correct using the (clipped) mean of good data",
433  "MEDIAN": "Correct using the median of the good data",
434  },
435  )
436 
437  # Overscan subtraction configuration.
438  doOverscan = pexConfig.Field(
439  dtype=bool,
440  doc="Do overscan subtraction?",
441  default=True,
442  )
443  overscan = pexConfig.ConfigurableField(
444  target=OverscanCorrectionTask,
445  doc="Overscan subtraction task for image segments.",
446  )
447 
448  overscanFitType = pexConfig.ChoiceField(
449  dtype=str,
450  doc="The method for fitting the overscan bias level.",
451  default='MEDIAN',
452  allowed={
453  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
454  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
455  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
456  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
457  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
458  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
459  "MEAN": "Correct using the mean of the overscan region",
460  "MEANCLIP": "Correct using a clipped mean of the overscan region",
461  "MEDIAN": "Correct using the median of the overscan region",
462  "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
463  },
464  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
465  " This option will no longer be used, and will be removed after v20.")
466  )
467  overscanOrder = pexConfig.Field(
468  dtype=int,
469  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, "
470  "or number of spline knots if overscan fit type is a spline."),
471  default=1,
472  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
473  " This option will no longer be used, and will be removed after v20.")
474  )
475  overscanNumSigmaClip = pexConfig.Field(
476  dtype=float,
477  doc="Rejection threshold (sigma) for collapsing overscan before fit",
478  default=3.0,
479  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
480  " This option will no longer be used, and will be removed after v20.")
481  )
482  overscanIsInt = pexConfig.Field(
483  dtype=bool,
484  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
485  " and overscan.FitType=MEDIAN_PER_ROW.",
486  default=True,
487  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
488  " This option will no longer be used, and will be removed after v20.")
489  )
490  # These options do not get deprecated, as they define how we slice up the image data.
491  overscanNumLeadingColumnsToSkip = pexConfig.Field(
492  dtype=int,
493  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
494  default=0,
495  )
496  overscanNumTrailingColumnsToSkip = pexConfig.Field(
497  dtype=int,
498  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
499  default=0,
500  )
501  overscanMaxDev = pexConfig.Field(
502  dtype=float,
503  doc="Maximum deviation from the median for overscan",
504  default=1000.0, check=lambda x: x > 0
505  )
506  overscanBiasJump = pexConfig.Field(
507  dtype=bool,
508  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
509  default=False,
510  )
511  overscanBiasJumpKeyword = pexConfig.Field(
512  dtype=str,
513  doc="Header keyword containing information about devices.",
514  default="NO_SUCH_KEY",
515  )
516  overscanBiasJumpDevices = pexConfig.ListField(
517  dtype=str,
518  doc="List of devices that need piecewise overscan correction.",
519  default=(),
520  )
521  overscanBiasJumpLocation = pexConfig.Field(
522  dtype=int,
523  doc="Location of bias jump along y-axis.",
524  default=0,
525  )
526 
527  # Amplifier to CCD assembly configuration
528  doAssembleCcd = pexConfig.Field(
529  dtype=bool,
530  default=True,
531  doc="Assemble amp-level exposures into a ccd-level exposure?"
532  )
533  assembleCcd = pexConfig.ConfigurableField(
534  target=AssembleCcdTask,
535  doc="CCD assembly task",
536  )
537 
538  # General calibration configuration.
539  doAssembleIsrExposures = pexConfig.Field(
540  dtype=bool,
541  default=False,
542  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
543  )
544  doTrimToMatchCalib = pexConfig.Field(
545  dtype=bool,
546  default=False,
547  doc="Trim raw data to match calibration bounding boxes?"
548  )
549 
550  # Bias subtraction.
551  doBias = pexConfig.Field(
552  dtype=bool,
553  doc="Apply bias frame correction?",
554  default=True,
555  )
556  biasDataProductName = pexConfig.Field(
557  dtype=str,
558  doc="Name of the bias data product",
559  default="bias",
560  )
561  doBiasBeforeOverscan = pexConfig.Field(
562  dtype=bool,
563  doc="Reverse order of overscan and bias correction.",
564  default=False
565  )
566 
567  # Variance construction
568  doVariance = pexConfig.Field(
569  dtype=bool,
570  doc="Calculate variance?",
571  default=True
572  )
573  gain = pexConfig.Field(
574  dtype=float,
575  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
576  default=float("NaN"),
577  )
578  readNoise = pexConfig.Field(
579  dtype=float,
580  doc="The read noise to use if no Detector is present in the Exposure",
581  default=0.0,
582  )
583  doEmpiricalReadNoise = pexConfig.Field(
584  dtype=bool,
585  default=False,
586  doc="Calculate empirical read noise instead of value from AmpInfo data?"
587  )
588  usePtcReadNoise = pexConfig.Field(
589  dtype=bool,
590  default=False,
591  doc="Use readnoise values from the Photon Transfer Curve?"
592  )
593  # Linearization.
594  doLinearize = pexConfig.Field(
595  dtype=bool,
596  doc="Correct for nonlinearity of the detector's response?",
597  default=True,
598  )
599 
600  # Crosstalk.
601  doCrosstalk = pexConfig.Field(
602  dtype=bool,
603  doc="Apply intra-CCD crosstalk correction?",
604  default=False,
605  )
606  doCrosstalkBeforeAssemble = pexConfig.Field(
607  dtype=bool,
608  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
609  default=False,
610  )
611  crosstalk = pexConfig.ConfigurableField(
612  target=CrosstalkTask,
613  doc="Intra-CCD crosstalk correction",
614  )
615 
616  # Masking options.
617  doDefect = pexConfig.Field(
618  dtype=bool,
619  doc="Apply correction for CCD defects, e.g. hot pixels?",
620  default=True,
621  )
622  doNanMasking = pexConfig.Field(
623  dtype=bool,
624  doc="Mask non-finite (NAN, inf) pixels?",
625  default=True,
626  )
627  doWidenSaturationTrails = pexConfig.Field(
628  dtype=bool,
629  doc="Widen bleed trails based on their width?",
630  default=True
631  )
632 
633  # Brighter-Fatter correction.
634  doBrighterFatter = pexConfig.Field(
635  dtype=bool,
636  default=False,
637  doc="Apply the brighter-fatter correction?"
638  )
639  brighterFatterLevel = pexConfig.ChoiceField(
640  dtype=str,
641  default="DETECTOR",
642  doc="The level at which to correct for brighter-fatter.",
643  allowed={
644  "AMP": "Every amplifier treated separately.",
645  "DETECTOR": "One kernel per detector",
646  }
647  )
648  brighterFatterMaxIter = pexConfig.Field(
649  dtype=int,
650  default=10,
651  doc="Maximum number of iterations for the brighter-fatter correction"
652  )
653  brighterFatterThreshold = pexConfig.Field(
654  dtype=float,
655  default=1000,
656  doc="Threshold used to stop iterating the brighter-fatter correction. It is the "
657  "absolute value of the difference between the current corrected image and the one "
658  "from the previous iteration summed over all the pixels."
659  )
660  brighterFatterApplyGain = pexConfig.Field(
661  dtype=bool,
662  default=True,
663  doc="Should the gain be applied when applying the brighter-fatter correction?"
664  )
665  brighterFatterMaskListToInterpolate = pexConfig.ListField(
666  dtype=str,
667  doc="List of mask planes that should be interpolated over when applying the brighter-fatter "
668  "correction.",
669  default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"],
670  )
671  brighterFatterMaskGrowSize = pexConfig.Field(
672  dtype=int,
673  default=0,
674  doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
675  "when brighter-fatter correction is applied."
676  )
677 
678  # Dark subtraction.
679  doDark = pexConfig.Field(
680  dtype=bool,
681  doc="Apply dark frame correction?",
682  default=True,
683  )
684  darkDataProductName = pexConfig.Field(
685  dtype=str,
686  doc="Name of the dark data product",
687  default="dark",
688  )
689 
690  # Camera-specific stray light removal.
691  doStrayLight = pexConfig.Field(
692  dtype=bool,
693  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
694  default=False,
695  )
696  strayLight = pexConfig.ConfigurableField(
697  target=StrayLightTask,
698  doc="y-band stray light correction"
699  )
700 
701  # Flat correction.
702  doFlat = pexConfig.Field(
703  dtype=bool,
704  doc="Apply flat field correction?",
705  default=True,
706  )
707  flatDataProductName = pexConfig.Field(
708  dtype=str,
709  doc="Name of the flat data product",
710  default="flat",
711  )
712  flatScalingType = pexConfig.ChoiceField(
713  dtype=str,
714  doc="The method for scaling the flat on the fly.",
715  default='USER',
716  allowed={
717  "USER": "Scale by flatUserScale",
718  "MEAN": "Scale by the inverse of the mean",
719  "MEDIAN": "Scale by the inverse of the median",
720  },
721  )
722  flatUserScale = pexConfig.Field(
723  dtype=float,
724  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
725  default=1.0,
726  )
727  doTweakFlat = pexConfig.Field(
728  dtype=bool,
729  doc="Tweak flats to match observed amplifier ratios?",
730  default=False
731  )
732 
733  # Amplifier normalization based on gains instead of using flats configuration.
734  doApplyGains = pexConfig.Field(
735  dtype=bool,
736  doc="Correct the amplifiers for their gains instead of applying flat correction",
737  default=False,
738  )
739  usePtcGains = pexConfig.Field(
740  dtype=bool,
741  doc="Use the gain values from the Photon Transfer Curve?",
742  default=False,
743  )
744  normalizeGains = pexConfig.Field(
745  dtype=bool,
746  doc="Normalize all the amplifiers in each CCD to have the same median value.",
747  default=False,
748  )
749 
750  # Fringe correction.
751  doFringe = pexConfig.Field(
752  dtype=bool,
753  doc="Apply fringe correction?",
754  default=True,
755  )
756  fringe = pexConfig.ConfigurableField(
757  target=FringeTask,
758  doc="Fringe subtraction task",
759  )
760  fringeAfterFlat = pexConfig.Field(
761  dtype=bool,
762  doc="Do fringe subtraction after flat-fielding?",
763  default=True,
764  )
765 
766  # Initial CCD-level background statistics options.
767  doMeasureBackground = pexConfig.Field(
768  dtype=bool,
769  doc="Measure the background level on the reduced image?",
770  default=False,
771  )
772 
773  # Camera-specific masking configuration.
774  doCameraSpecificMasking = pexConfig.Field(
775  dtype=bool,
776  doc="Mask camera-specific bad regions?",
777  default=False,
778  )
779  masking = pexConfig.ConfigurableField(
780  target=MaskingTask,
781  doc="Masking task."
782  )
783 
784  # Interpolation options.
785 
786  doInterpolate = pexConfig.Field(
787  dtype=bool,
788  doc="Interpolate masked pixels?",
789  default=True,
790  )
791  doSaturationInterpolation = pexConfig.Field(
792  dtype=bool,
793  doc="Perform interpolation over pixels masked as saturated?"
794  " NB: This is independent of doSaturation; if that is False this plane"
795  " will likely be blank, resulting in a no-op here.",
796  default=True,
797  )
798  doNanInterpolation = pexConfig.Field(
799  dtype=bool,
800  doc="Perform interpolation over pixels masked as NaN?"
801  " NB: This is independent of doNanMasking; if that is False this plane"
802  " will likely be blank, resulting in a no-op here.",
803  default=True,
804  )
805  doNanInterpAfterFlat = pexConfig.Field(
806  dtype=bool,
807  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
808  "also have to interpolate them before flat-fielding."),
809  default=False,
810  )
811  maskListToInterpolate = pexConfig.ListField(
812  dtype=str,
813  doc="List of mask planes that should be interpolated.",
814  default=['SAT', 'BAD'],
815  )
816  doSaveInterpPixels = pexConfig.Field(
817  dtype=bool,
818  doc="Save a copy of the pre-interpolated pixel values?",
819  default=False,
820  )
821 
822  # Default photometric calibration options.
823  fluxMag0T1 = pexConfig.DictField(
824  keytype=str,
825  itemtype=float,
826  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
827  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
828  ))
829  )
830  defaultFluxMag0T1 = pexConfig.Field(
831  dtype=float,
832  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
833  default=pow(10.0, 0.4*28.0)
834  )
835 
836  # Vignette correction configuration.
837  doVignette = pexConfig.Field(
838  dtype=bool,
839  doc="Apply vignetting parameters?",
840  default=False,
841  )
842  vignette = pexConfig.ConfigurableField(
843  target=VignetteTask,
844  doc="Vignetting task.",
845  )
846 
847  # Transmission curve configuration.
848  doAttachTransmissionCurve = pexConfig.Field(
849  dtype=bool,
850  default=False,
851  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
852  )
853  doUseOpticsTransmission = pexConfig.Field(
854  dtype=bool,
855  default=True,
856  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
857  )
858  doUseFilterTransmission = pexConfig.Field(
859  dtype=bool,
860  default=True,
861  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
862  )
863  doUseSensorTransmission = pexConfig.Field(
864  dtype=bool,
865  default=True,
866  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
867  )
868  doUseAtmosphereTransmission = pexConfig.Field(
869  dtype=bool,
870  default=True,
871  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
872  )
873 
874  # Illumination correction.
875  doIlluminationCorrection = pexConfig.Field(
876  dtype=bool,
877  default=False,
878  doc="Perform illumination correction?"
879  )
880  illuminationCorrectionDataProductName = pexConfig.Field(
881  dtype=str,
882  doc="Name of the illumination correction data product.",
883  default="illumcor",
884  )
885  illumScale = pexConfig.Field(
886  dtype=float,
887  doc="Scale factor for the illumination correction.",
888  default=1.0,
889  )
890  illumFilters = pexConfig.ListField(
891  dtype=str,
892  default=[],
893  doc="Only perform illumination correction for these filters."
894  )
895 
896  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
897  doWrite = pexConfig.Field(
898  dtype=bool,
899  doc="Persist postISRCCD?",
900  default=True,
901  )
902 
903  def validate(self):
904  super().validate()
905  if self.doFlatdoFlat and self.doApplyGainsdoApplyGains:
906  raise ValueError("You may not specify both doFlat and doApplyGains")
907  if self.doBiasBeforeOverscandoBiasBeforeOverscan and self.doTrimToMatchCalibdoTrimToMatchCalib:
908  raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
909  if self.doSaturationInterpolationdoSaturationInterpolation and self.saturatedMaskNamesaturatedMaskName not in self.maskListToInterpolatemaskListToInterpolate:
910  self.maskListToInterpolatemaskListToInterpolate.append(self.saturatedMaskNamesaturatedMaskName)
911  if not self.doSaturationInterpolationdoSaturationInterpolation and self.saturatedMaskNamesaturatedMaskName in self.maskListToInterpolatemaskListToInterpolate:
912  self.maskListToInterpolatemaskListToInterpolate.remove(self.saturatedMaskNamesaturatedMaskName)
913  if self.doNanInterpolationdoNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolatemaskListToInterpolate:
914  self.maskListToInterpolatemaskListToInterpolate.append("UNMASKEDNAN")
915 
916 
917 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
918  """Apply common instrument signature correction algorithms to a raw frame.
919 
920  The process for correcting imaging data is very similar from
921  camera to camera. This task provides a vanilla implementation of
922  doing these corrections, including the ability to turn certain
923  corrections off if they are not needed. The inputs to the primary
924  method, `run()`, are a raw exposure to be corrected and the
925  calibration data products. The raw input is a single chip sized
926  mosaic of all amps including overscans and other non-science
927  pixels. The method `runDataRef()` identifies and defines the
928  calibration data products, and is intended for use by a
929  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
930  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
931  subclassed for different camera, although the most camera specific
932  methods have been split into subtasks that can be redirected
933  appropriately.
934 
935  The __init__ method sets up the subtasks for ISR processing, using
936  the defaults from `lsst.ip.isr`.
937 
938  Parameters
939  ----------
940  args : `list`
941  Positional arguments passed to the Task constructor. None used at this time.
942  kwargs : `dict`, optional
943  Keyword arguments passed on to the Task constructor. None used at this time.
944  """
945  ConfigClass = IsrTaskConfig
946  _DefaultName = "isr"
947 
948  def __init__(self, **kwargs):
949  super().__init__(**kwargs)
950  self.makeSubtask("assembleCcd")
951  self.makeSubtask("crosstalk")
952  self.makeSubtask("strayLight")
953  self.makeSubtask("fringe")
954  self.makeSubtask("masking")
955  self.makeSubtask("overscan")
956  self.makeSubtask("vignette")
957 
958  def runQuantum(self, butlerQC, inputRefs, outputRefs):
959  inputs = butlerQC.get(inputRefs)
960 
961  try:
962  inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
963  except Exception as e:
964  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
965  (inputRefs, e))
966 
967  inputs['isGen3'] = True
968 
969  detector = inputs['ccdExposure'].getDetector()
970 
971  if self.config.doCrosstalk is True:
972  # Crosstalk sources need to be defined by the pipeline
973  # yaml if they exist.
974  if 'crosstalk' in inputs and inputs['crosstalk'] is not None:
975  if not isinstance(inputs['crosstalk'], CrosstalkCalib):
976  inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk'])
977  else:
978  coeffVector = (self.config.crosstalk.crosstalkValues
979  if self.config.crosstalk.useConfigCoefficients else None)
980  crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
981  inputs['crosstalk'] = crosstalkCalib
982  if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0:
983  if 'crosstalkSources' not in inputs:
984  self.log.warn("No crosstalkSources found for chip with interChip terms!")
985 
986  if self.doLinearizedoLinearize(detector) is True:
987  if 'linearizer' in inputs:
988  if isinstance(inputs['linearizer'], dict):
989  linearizer = linearize.Linearizer(detector=detector, log=self.log)
990  linearizer.fromYaml(inputs['linearizer'])
991  self.log.warn("Dictionary linearizers will be deprecated in DM-28741.")
992  elif isinstance(inputs['linearizer'], numpy.ndarray):
993  linearizer = linearize.Linearizer(table=inputs.get('linearizer', None),
994  detector=detector,
995  log=self.log)
996  self.log.warn("Bare lookup table linearizers will be deprecated in DM-28741.")
997  else:
998  linearizer = inputs['linearizer']
999  linearizer.log = self.log
1000  inputs['linearizer'] = linearizer
1001  else:
1002  inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log)
1003  self.log.warn("Constructing linearizer from cameraGeom information.")
1004 
1005  if self.config.doDefect is True:
1006  if "defects" in inputs and inputs['defects'] is not None:
1007  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
1008  # masking expects a list of defects defined by their bounding box
1009  if not isinstance(inputs["defects"], Defects):
1010  inputs["defects"] = Defects.fromTable(inputs["defects"])
1011 
1012  # Load the correct style of brighter-fatter kernel, and repack
1013  # the information as a numpy array.
1014  if self.config.doBrighterFatter:
1015  brighterFatterKernel = inputs.pop('newBFKernel', None)
1016  if brighterFatterKernel is None:
1017  brighterFatterKernel = inputs.get('bfKernel', None)
1018 
1019  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1020  # This is a ISR calib kernel
1021  detName = detector.getName()
1022  level = brighterFatterKernel.level
1023 
1024  # This is expected to be a dictionary of amp-wise gains.
1025  inputs['bfGains'] = brighterFatterKernel.gain
1026  if self.config.brighterFatterLevel == 'DETECTOR':
1027  if level == 'DETECTOR':
1028  if detName in brighterFatterKernel.detKernels:
1029  inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1030  else:
1031  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1032  elif level == 'AMP':
1033  self.log.warn("Making DETECTOR level kernel from AMP based brighter fatter kernels.")
1034  brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1035  inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1036  elif self.config.brighterFatterLevel == 'AMP':
1037  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1038 
1039  if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
1040  expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
1041  inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
1042  expId=expId,
1043  assembler=self.assembleCcd
1044  if self.config.doAssembleIsrExposures else None)
1045  else:
1046  inputs['fringes'] = pipeBase.Struct(fringes=None)
1047 
1048  if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
1049  if 'strayLightData' not in inputs:
1050  inputs['strayLightData'] = None
1051 
1052  outputs = self.runrun(**inputs)
1053  butlerQC.put(outputs, outputRefs)
1054 
1055  def readIsrData(self, dataRef, rawExposure):
1056  """Retrieve necessary frames for instrument signature removal.
1057 
1058  Pre-fetching all required ISR data products limits the IO
1059  required by the ISR. Any conflict between the calibration data
1060  available and that needed for ISR is also detected prior to
1061  doing processing, allowing it to fail quickly.
1062 
1063  Parameters
1064  ----------
1065  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1066  Butler reference of the detector data to be processed
1067  rawExposure : `afw.image.Exposure`
1068  The raw exposure that will later be corrected with the
1069  retrieved calibration data; should not be modified in this
1070  method.
1071 
1072  Returns
1073  -------
1074  result : `lsst.pipe.base.Struct`
1075  Result struct with components (which may be `None`):
1076  - ``bias``: bias calibration frame (`afw.image.Exposure`)
1077  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
1078  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1079  - ``dark``: dark calibration frame (`afw.image.Exposure`)
1080  - ``flat``: flat calibration frame (`afw.image.Exposure`)
1081  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1082  - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1083  - ``fringes``: `lsst.pipe.base.Struct` with components:
1084  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1085  - ``seed``: random seed derived from the ccdExposureId for random
1086  number generator (`uint32`).
1087  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1088  A ``TransmissionCurve`` that represents the throughput of the optics,
1089  to be evaluated in focal-plane coordinates.
1090  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1091  A ``TransmissionCurve`` that represents the throughput of the filter
1092  itself, to be evaluated in focal-plane coordinates.
1093  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1094  A ``TransmissionCurve`` that represents the throughput of the sensor
1095  itself, to be evaluated in post-assembly trimmed detector coordinates.
1096  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1097  A ``TransmissionCurve`` that represents the throughput of the
1098  atmosphere, assumed to be spatially constant.
1099  - ``strayLightData`` : `object`
1100  An opaque object containing calibration information for
1101  stray-light correction. If `None`, no correction will be
1102  performed.
1103  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1104 
1105  Raises
1106  ------
1107  NotImplementedError :
1108  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1109  """
1110  try:
1111  dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1112  dateObs = dateObs.toPython().isoformat()
1113  except RuntimeError:
1114  self.log.warn("Unable to identify dateObs for rawExposure.")
1115  dateObs = None
1116 
1117  ccd = rawExposure.getDetector()
1118  filterLabel = rawExposure.getFilterLabel()
1119  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1120  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
1121  biasExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.biasDataProductName)
1122  if self.config.doBias else None)
1123  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
1124  linearizer = (dataRef.get("linearizer", immediate=True)
1125  if self.doLinearizedoLinearize(ccd) else None)
1126  if linearizer is not None and not isinstance(linearizer, numpy.ndarray):
1127  linearizer.log = self.log
1128  if isinstance(linearizer, numpy.ndarray):
1129  linearizer = linearize.Linearizer(table=linearizer, detector=ccd)
1130 
1131  crosstalkCalib = None
1132  if self.config.doCrosstalk:
1133  try:
1134  crosstalkCalib = dataRef.get("crosstalk", immediate=True)
1135  except NoResults:
1136  coeffVector = (self.config.crosstalk.crosstalkValues
1137  if self.config.crosstalk.useConfigCoefficients else None)
1138  crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1139  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1140  if self.config.doCrosstalk else None)
1141 
1142  darkExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.darkDataProductName)
1143  if self.config.doDark else None)
1144  flatExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.flatDataProductName,
1145  dateObs=dateObs)
1146  if self.config.doFlat else None)
1147 
1148  brighterFatterKernel = None
1149  brighterFatterGains = None
1150  if self.config.doBrighterFatter is True:
1151  try:
1152  # Use the new-style cp_pipe version of the kernel if it exists
1153  # If using a new-style kernel, always use the self-consistent
1154  # gains, i.e. the ones inside the kernel object itself
1155  brighterFatterKernel = dataRef.get("brighterFatterKernel")
1156  brighterFatterGains = brighterFatterKernel.gain
1157  self.log.info("New style brighter-fatter kernel (brighterFatterKernel) loaded")
1158  except NoResults:
1159  try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
1160  brighterFatterKernel = dataRef.get("bfKernel")
1161  self.log.info("Old style brighter-fatter kernel (bfKernel) loaded")
1162  except NoResults:
1163  brighterFatterKernel = None
1164  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1165  # If the kernel is not an ndarray, it's the cp_pipe version
1166  # so extract the kernel for this detector, or raise an error
1167  if self.config.brighterFatterLevel == 'DETECTOR':
1168  if brighterFatterKernel.detKernels:
1169  brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()]
1170  else:
1171  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1172  else:
1173  # TODO DM-15631 for implementing this
1174  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1175 
1176  defectList = (dataRef.get("defects")
1177  if self.config.doDefect else None)
1178  expId = rawExposure.getInfo().getVisitInfo().getExposureId()
1179  fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1180  if self.config.doAssembleIsrExposures else None)
1181  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1182  else pipeBase.Struct(fringes=None))
1183 
1184  if self.config.doAttachTransmissionCurve:
1185  opticsTransmission = (dataRef.get("transmission_optics")
1186  if self.config.doUseOpticsTransmission else None)
1187  filterTransmission = (dataRef.get("transmission_filter")
1188  if self.config.doUseFilterTransmission else None)
1189  sensorTransmission = (dataRef.get("transmission_sensor")
1190  if self.config.doUseSensorTransmission else None)
1191  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1192  if self.config.doUseAtmosphereTransmission else None)
1193  else:
1194  opticsTransmission = None
1195  filterTransmission = None
1196  sensorTransmission = None
1197  atmosphereTransmission = None
1198 
1199  if self.config.doStrayLight:
1200  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1201  else:
1202  strayLightData = None
1203 
1204  illumMaskedImage = (self.getIsrExposuregetIsrExposure(dataRef,
1205  self.config.illuminationCorrectionDataProductName).getMaskedImage()
1206  if (self.config.doIlluminationCorrection
1207  and physicalFilter in self.config.illumFilters)
1208  else None)
1209 
1210  # Struct should include only kwargs to run()
1211  return pipeBase.Struct(bias=biasExposure,
1212  linearizer=linearizer,
1213  crosstalk=crosstalkCalib,
1214  crosstalkSources=crosstalkSources,
1215  dark=darkExposure,
1216  flat=flatExposure,
1217  bfKernel=brighterFatterKernel,
1218  bfGains=brighterFatterGains,
1219  defects=defectList,
1220  fringes=fringeStruct,
1221  opticsTransmission=opticsTransmission,
1222  filterTransmission=filterTransmission,
1223  sensorTransmission=sensorTransmission,
1224  atmosphereTransmission=atmosphereTransmission,
1225  strayLightData=strayLightData,
1226  illumMaskedImage=illumMaskedImage
1227  )
1228 
1229  @pipeBase.timeMethod
1230  def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1231  crosstalk=None, crosstalkSources=None,
1232  dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1233  fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1234  sensorTransmission=None, atmosphereTransmission=None,
1235  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1236  isGen3=False,
1237  ):
1238  """Perform instrument signature removal on an exposure.
1239 
1240  Steps included in the ISR processing, in order performed, are:
1241  - saturation and suspect pixel masking
1242  - overscan subtraction
1243  - CCD assembly of individual amplifiers
1244  - bias subtraction
1245  - variance image construction
1246  - linearization of non-linear response
1247  - crosstalk masking
1248  - brighter-fatter correction
1249  - dark subtraction
1250  - fringe correction
1251  - stray light subtraction
1252  - flat correction
1253  - masking of known defects and camera specific features
1254  - vignette calculation
1255  - appending transmission curve and distortion model
1256 
1257  Parameters
1258  ----------
1259  ccdExposure : `lsst.afw.image.Exposure`
1260  The raw exposure that is to be run through ISR. The
1261  exposure is modified by this method.
1262  camera : `lsst.afw.cameraGeom.Camera`, optional
1263  The camera geometry for this exposure. Required if ``isGen3`` is
1264  `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1265  ``flat`` does not have an associated detector.
1266  bias : `lsst.afw.image.Exposure`, optional
1267  Bias calibration frame.
1268  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1269  Functor for linearization.
1270  crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1271  Calibration for crosstalk.
1272  crosstalkSources : `list`, optional
1273  List of possible crosstalk sources.
1274  dark : `lsst.afw.image.Exposure`, optional
1275  Dark calibration frame.
1276  flat : `lsst.afw.image.Exposure`, optional
1277  Flat calibration frame.
1278  ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
1279  Photon transfer curve dataset, with, e.g., gains
1280  and read noise.
1281  bfKernel : `numpy.ndarray`, optional
1282  Brighter-fatter kernel.
1283  bfGains : `dict` of `float`, optional
1284  Gains used to override the detector's nominal gains for the
1285  brighter-fatter correction. A dict keyed by amplifier name for
1286  the detector in question.
1287  defects : `lsst.ip.isr.Defects`, optional
1288  List of defects.
1289  fringes : `lsst.pipe.base.Struct`, optional
1290  Struct containing the fringe correction data, with
1291  elements:
1292  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1293  - ``seed``: random seed derived from the ccdExposureId for random
1294  number generator (`uint32`)
1295  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1296  A ``TransmissionCurve`` that represents the throughput of the optics,
1297  to be evaluated in focal-plane coordinates.
1298  filterTransmission : `lsst.afw.image.TransmissionCurve`
1299  A ``TransmissionCurve`` that represents the throughput of the filter
1300  itself, to be evaluated in focal-plane coordinates.
1301  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1302  A ``TransmissionCurve`` that represents the throughput of the sensor
1303  itself, to be evaluated in post-assembly trimmed detector coordinates.
1304  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1305  A ``TransmissionCurve`` that represents the throughput of the
1306  atmosphere, assumed to be spatially constant.
1307  detectorNum : `int`, optional
1308  The integer number for the detector to process.
1309  isGen3 : bool, optional
1310  Flag this call to run() as using the Gen3 butler environment.
1311  strayLightData : `object`, optional
1312  Opaque object containing calibration information for stray-light
1313  correction. If `None`, no correction will be performed.
1314  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1315  Illumination correction image.
1316 
1317  Returns
1318  -------
1319  result : `lsst.pipe.base.Struct`
1320  Result struct with component:
1321  - ``exposure`` : `afw.image.Exposure`
1322  The fully ISR corrected exposure.
1323  - ``outputExposure`` : `afw.image.Exposure`
1324  An alias for `exposure`
1325  - ``ossThumb`` : `numpy.ndarray`
1326  Thumbnail image of the exposure after overscan subtraction.
1327  - ``flattenedThumb`` : `numpy.ndarray`
1328  Thumbnail image of the exposure after flat-field correction.
1329 
1330  Raises
1331  ------
1332  RuntimeError
1333  Raised if a configuration option is set to True, but the
1334  required calibration data has not been specified.
1335 
1336  Notes
1337  -----
1338  The current processed exposure can be viewed by setting the
1339  appropriate lsstDebug entries in the `debug.display`
1340  dictionary. The names of these entries correspond to some of
1341  the IsrTaskConfig Boolean options, with the value denoting the
1342  frame to use. The exposure is shown inside the matching
1343  option check and after the processing of that step has
1344  finished. The steps with debug points are:
1345 
1346  doAssembleCcd
1347  doBias
1348  doCrosstalk
1349  doBrighterFatter
1350  doDark
1351  doFringe
1352  doStrayLight
1353  doFlat
1354 
1355  In addition, setting the "postISRCCD" entry displays the
1356  exposure after all ISR processing has finished.
1357 
1358  """
1359 
1360  if isGen3 is True:
1361  # Gen3 currently cannot automatically do configuration overrides.
1362  # DM-15257 looks to discuss this issue.
1363  # Configure input exposures;
1364  if detectorNum is None:
1365  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1366 
1367  ccdExposure = self.ensureExposureensureExposure(ccdExposure, camera, detectorNum)
1368  bias = self.ensureExposureensureExposure(bias, camera, detectorNum)
1369  dark = self.ensureExposureensureExposure(dark, camera, detectorNum)
1370  flat = self.ensureExposureensureExposure(flat, camera, detectorNum)
1371  else:
1372  if isinstance(ccdExposure, ButlerDataRef):
1373  return self.runDataRefrunDataRef(ccdExposure)
1374 
1375  ccd = ccdExposure.getDetector()
1376  filterLabel = ccdExposure.getFilterLabel()
1377  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1378 
1379  if not ccd:
1380  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1381  ccd = [FakeAmp(ccdExposure, self.config)]
1382 
1383  # Validate Input
1384  if self.config.doBias and bias is None:
1385  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1386  if self.doLinearizedoLinearize(ccd) and linearizer is None:
1387  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1388  if self.config.doBrighterFatter and bfKernel is None:
1389  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1390  if self.config.doDark and dark is None:
1391  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1392  if self.config.doFlat and flat is None:
1393  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1394  if self.config.doDefect and defects is None:
1395  raise RuntimeError("Must supply defects if config.doDefect=True.")
1396  if (self.config.doFringe and physicalFilter in self.fringe.config.filters
1397  and fringes.fringes is None):
1398  # The `fringes` object needs to be a pipeBase.Struct, as
1399  # we use it as a `dict` for the parameters of
1400  # `FringeTask.run()`. The `fringes.fringes` `list` may
1401  # not be `None` if `doFringe=True`. Otherwise, raise.
1402  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1403  if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters
1404  and illumMaskedImage is None):
1405  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1406 
1407  # Begin ISR processing.
1408  if self.config.doConvertIntToFloat:
1409  self.log.info("Converting exposure to floating point values.")
1410  ccdExposure = self.convertIntToFloatconvertIntToFloat(ccdExposure)
1411 
1412  if self.config.doBias and self.config.doBiasBeforeOverscan:
1413  self.log.info("Applying bias correction.")
1414  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1415  trimToFit=self.config.doTrimToMatchCalib)
1416  self.debugViewdebugView(ccdExposure, "doBias")
1417 
1418  # Amplifier level processing.
1419  overscans = []
1420  for amp in ccd:
1421  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1422  if ccdExposure.getBBox().contains(amp.getBBox()):
1423  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1424  badAmp = self.maskAmplifiermaskAmplifier(ccdExposure, amp, defects)
1425 
1426  if self.config.doOverscan and not badAmp:
1427  # Overscan correction on amp-by-amp basis.
1428  overscanResults = self.overscanCorrectionoverscanCorrection(ccdExposure, amp)
1429  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1430  if overscanResults is not None and \
1431  self.config.qa is not None and self.config.qa.saveStats is True:
1432  if isinstance(overscanResults.overscanFit, float):
1433  qaMedian = overscanResults.overscanFit
1434  qaStdev = float("NaN")
1435  else:
1436  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1437  afwMath.MEDIAN | afwMath.STDEVCLIP)
1438  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1439  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1440 
1441  self.metadata.set(f"FIT MEDIAN {amp.getName()}", qaMedian)
1442  self.metadata.set(f"FIT STDEV {amp.getName()}", qaStdev)
1443  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1444  amp.getName(), qaMedian, qaStdev)
1445 
1446  # Residuals after overscan correction
1447  qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1448  afwMath.MEDIAN | afwMath.STDEVCLIP)
1449  qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1450  qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1451 
1452  self.metadata.set(f"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter)
1453  self.metadata.set(f"RESIDUAL STDEV {amp.getName()}", qaStdevAfter)
1454  self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f",
1455  amp.getName(), qaMedianAfter, qaStdevAfter)
1456 
1457  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1458  else:
1459  if badAmp:
1460  self.log.warn("Amplifier %s is bad.", amp.getName())
1461  overscanResults = None
1462 
1463  overscans.append(overscanResults if overscanResults is not None else None)
1464  else:
1465  self.log.info("Skipped OSCAN for %s.", amp.getName())
1466 
1467  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1468  self.log.info("Applying crosstalk correction.")
1469  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1470  crosstalkSources=crosstalkSources, camera=camera)
1471  self.debugViewdebugView(ccdExposure, "doCrosstalk")
1472 
1473  if self.config.doAssembleCcd:
1474  self.log.info("Assembling CCD from amplifiers.")
1475  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1476 
1477  if self.config.expectWcs and not ccdExposure.getWcs():
1478  self.log.warn("No WCS found in input exposure.")
1479  self.debugViewdebugView(ccdExposure, "doAssembleCcd")
1480 
1481  ossThumb = None
1482  if self.config.qa.doThumbnailOss:
1483  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1484 
1485  if self.config.doBias and not self.config.doBiasBeforeOverscan:
1486  self.log.info("Applying bias correction.")
1487  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1488  trimToFit=self.config.doTrimToMatchCalib)
1489  self.debugViewdebugView(ccdExposure, "doBias")
1490 
1491  if self.config.doVariance:
1492  for amp, overscanResults in zip(ccd, overscans):
1493  if ccdExposure.getBBox().contains(amp.getBBox()):
1494  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1495  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1496  if overscanResults is not None:
1497  self.updateVarianceupdateVariance(ampExposure, amp,
1498  overscanImage=overscanResults.overscanImage,
1499  ptcDataset=ptc)
1500  else:
1501  self.updateVarianceupdateVariance(ampExposure, amp,
1502  overscanImage=None,
1503  ptcDataset=ptc)
1504  if self.config.qa is not None and self.config.qa.saveStats is True:
1505  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1506  afwMath.MEDIAN | afwMath.STDEVCLIP)
1507  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1508  qaStats.getValue(afwMath.MEDIAN))
1509  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1510  qaStats.getValue(afwMath.STDEVCLIP))
1511  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1512  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1513  qaStats.getValue(afwMath.STDEVCLIP))
1514 
1515  if self.doLinearizedoLinearize(ccd):
1516  self.log.info("Applying linearizer.")
1517  linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1518  detector=ccd, log=self.log)
1519 
1520  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1521  self.log.info("Applying crosstalk correction.")
1522  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1523  crosstalkSources=crosstalkSources, isTrimmed=True)
1524  self.debugViewdebugView(ccdExposure, "doCrosstalk")
1525 
1526  # Masking block. Optionally mask known defects, NAN/inf pixels, widen trails, and do
1527  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1528  if self.config.doDefect:
1529  self.log.info("Masking defects.")
1530  self.maskDefectmaskDefect(ccdExposure, defects)
1531 
1532  if self.config.numEdgeSuspect > 0:
1533  self.log.info("Masking edges as SUSPECT.")
1534  self.maskEdgesmaskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1535  maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
1536 
1537  if self.config.doNanMasking:
1538  self.log.info("Masking non-finite (NAN, inf) value pixels.")
1539  self.maskNanmaskNan(ccdExposure)
1540 
1541  if self.config.doWidenSaturationTrails:
1542  self.log.info("Widening saturation trails.")
1543  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1544 
1545  if self.config.doCameraSpecificMasking:
1546  self.log.info("Masking regions for camera specific reasons.")
1547  self.masking.run(ccdExposure)
1548 
1549  if self.config.doBrighterFatter:
1550  # We need to apply flats and darks before we can interpolate, and we
1551  # need to interpolate before we do B-F, but we do B-F without the
1552  # flats and darks applied so we can work in units of electrons or holes.
1553  # This context manager applies and then removes the darks and flats.
1554  #
1555  # We also do not want to interpolate values here, so operate on temporary
1556  # images so we can apply only the BF-correction and roll back the
1557  # interpolation.
1558  interpExp = ccdExposure.clone()
1559  with self.flatContextflatContext(interpExp, flat, dark):
1560  isrFunctions.interpolateFromMask(
1561  maskedImage=interpExp.getMaskedImage(),
1562  fwhm=self.config.fwhm,
1563  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1564  maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1565  )
1566  bfExp = interpExp.clone()
1567 
1568  self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.",
1569  type(bfKernel), type(bfGains))
1570  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1571  self.config.brighterFatterMaxIter,
1572  self.config.brighterFatterThreshold,
1573  self.config.brighterFatterApplyGain,
1574  bfGains)
1575  if bfResults[1] == self.config.brighterFatterMaxIter:
1576  self.log.warn("Brighter-fatter correction did not converge, final difference %f.",
1577  bfResults[0])
1578  else:
1579  self.log.info("Finished brighter-fatter correction in %d iterations.",
1580  bfResults[1])
1581  image = ccdExposure.getMaskedImage().getImage()
1582  bfCorr = bfExp.getMaskedImage().getImage()
1583  bfCorr -= interpExp.getMaskedImage().getImage()
1584  image += bfCorr
1585 
1586  # Applying the brighter-fatter correction applies a
1587  # convolution to the science image. At the edges this
1588  # convolution may not have sufficient valid pixels to
1589  # produce a valid correction. Mark pixels within the size
1590  # of the brighter-fatter kernel as EDGE to warn of this
1591  # fact.
1592  self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1593  self.maskEdgesmaskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1594  maskPlane="EDGE")
1595 
1596  if self.config.brighterFatterMaskGrowSize > 0:
1597  self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1598  for maskPlane in self.config.brighterFatterMaskListToInterpolate:
1599  isrFunctions.growMasks(ccdExposure.getMask(),
1600  radius=self.config.brighterFatterMaskGrowSize,
1601  maskNameList=maskPlane,
1602  maskValue=maskPlane)
1603 
1604  self.debugViewdebugView(ccdExposure, "doBrighterFatter")
1605 
1606  if self.config.doDark:
1607  self.log.info("Applying dark correction.")
1608  self.darkCorrectiondarkCorrection(ccdExposure, dark)
1609  self.debugViewdebugView(ccdExposure, "doDark")
1610 
1611  if self.config.doFringe and not self.config.fringeAfterFlat:
1612  self.log.info("Applying fringe correction before flat.")
1613  self.fringe.run(ccdExposure, **fringes.getDict())
1614  self.debugViewdebugView(ccdExposure, "doFringe")
1615 
1616  if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1617  self.log.info("Checking strayLight correction.")
1618  self.strayLight.run(ccdExposure, strayLightData)
1619  self.debugViewdebugView(ccdExposure, "doStrayLight")
1620 
1621  if self.config.doFlat:
1622  self.log.info("Applying flat correction.")
1623  self.flatCorrectionflatCorrection(ccdExposure, flat)
1624  self.debugViewdebugView(ccdExposure, "doFlat")
1625 
1626  if self.config.doApplyGains:
1627  self.log.info("Applying gain correction instead of flat.")
1628  if self.config.usePtcGains:
1629  self.log.info("Using gains from the Photon Transfer Curve.")
1630  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1631  ptcGains=ptc.gain)
1632  else:
1633  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1634 
1635  if self.config.doFringe and self.config.fringeAfterFlat:
1636  self.log.info("Applying fringe correction after flat.")
1637  self.fringe.run(ccdExposure, **fringes.getDict())
1638 
1639  if self.config.doVignette:
1640  self.log.info("Constructing Vignette polygon.")
1641  self.vignettePolygonvignettePolygon = self.vignette.run(ccdExposure)
1642 
1643  if self.config.vignette.doWriteVignettePolygon:
1644  self.setValidPolygonIntersectsetValidPolygonIntersect(ccdExposure, self.vignettePolygonvignettePolygon)
1645 
1646  if self.config.doAttachTransmissionCurve:
1647  self.log.info("Adding transmission curves.")
1648  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1649  filterTransmission=filterTransmission,
1650  sensorTransmission=sensorTransmission,
1651  atmosphereTransmission=atmosphereTransmission)
1652 
1653  flattenedThumb = None
1654  if self.config.qa.doThumbnailFlattened:
1655  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1656 
1657  if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters:
1658  self.log.info("Performing illumination correction.")
1659  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1660  illumMaskedImage, illumScale=self.config.illumScale,
1661  trimToFit=self.config.doTrimToMatchCalib)
1662 
1663  preInterpExp = None
1664  if self.config.doSaveInterpPixels:
1665  preInterpExp = ccdExposure.clone()
1666 
1667  # Reset and interpolate bad pixels.
1668  #
1669  # Large contiguous bad regions (which should have the BAD mask
1670  # bit set) should have their values set to the image median.
1671  # This group should include defects and bad amplifiers. As the
1672  # area covered by these defects are large, there's little
1673  # reason to expect that interpolation would provide a more
1674  # useful value.
1675  #
1676  # Smaller defects can be safely interpolated after the larger
1677  # regions have had their pixel values reset. This ensures
1678  # that the remaining defects adjacent to bad amplifiers (as an
1679  # example) do not attempt to interpolate extreme values.
1680  if self.config.doSetBadRegions:
1681  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1682  if badPixelCount > 0:
1683  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1684 
1685  if self.config.doInterpolate:
1686  self.log.info("Interpolating masked pixels.")
1687  isrFunctions.interpolateFromMask(
1688  maskedImage=ccdExposure.getMaskedImage(),
1689  fwhm=self.config.fwhm,
1690  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1691  maskNameList=list(self.config.maskListToInterpolate)
1692  )
1693 
1694  self.roughZeroPointroughZeroPoint(ccdExposure)
1695 
1696  if self.config.doMeasureBackground:
1697  self.log.info("Measuring background level.")
1698  self.measureBackgroundmeasureBackground(ccdExposure, self.config.qa)
1699 
1700  if self.config.qa is not None and self.config.qa.saveStats is True:
1701  for amp in ccd:
1702  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1703  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1704  afwMath.MEDIAN | afwMath.STDEVCLIP)
1705  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1706  qaStats.getValue(afwMath.MEDIAN))
1707  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1708  qaStats.getValue(afwMath.STDEVCLIP))
1709  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1710  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1711  qaStats.getValue(afwMath.STDEVCLIP))
1712 
1713  self.debugViewdebugView(ccdExposure, "postISRCCD")
1714 
1715  return pipeBase.Struct(
1716  exposure=ccdExposure,
1717  ossThumb=ossThumb,
1718  flattenedThumb=flattenedThumb,
1719 
1720  preInterpExposure=preInterpExp,
1721  outputExposure=ccdExposure,
1722  outputOssThumbnail=ossThumb,
1723  outputFlattenedThumbnail=flattenedThumb,
1724  )
1725 
1726  @pipeBase.timeMethod
1727  def runDataRef(self, sensorRef):
1728  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1729 
1730  This method contains the `CmdLineTask` interface to the ISR
1731  processing. All IO is handled here, freeing the `run()` method
1732  to manage only pixel-level calculations. The steps performed
1733  are:
1734  - Read in necessary detrending/isr/calibration data.
1735  - Process raw exposure in `run()`.
1736  - Persist the ISR-corrected exposure as "postISRCCD" if
1737  config.doWrite=True.
1738 
1739  Parameters
1740  ----------
1741  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1742  DataRef of the detector data to be processed
1743 
1744  Returns
1745  -------
1746  result : `lsst.pipe.base.Struct`
1747  Result struct with component:
1748  - ``exposure`` : `afw.image.Exposure`
1749  The fully ISR corrected exposure.
1750 
1751  Raises
1752  ------
1753  RuntimeError
1754  Raised if a configuration option is set to True, but the
1755  required calibration data does not exist.
1756 
1757  """
1758  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1759 
1760  ccdExposure = sensorRef.get(self.config.datasetType)
1761 
1762  camera = sensorRef.get("camera")
1763  isrData = self.readIsrDatareadIsrData(sensorRef, ccdExposure)
1764 
1765  result = self.runrun(ccdExposure, camera=camera, **isrData.getDict())
1766 
1767  if self.config.doWrite:
1768  sensorRef.put(result.exposure, "postISRCCD")
1769  if result.preInterpExposure is not None:
1770  sensorRef.put(result.preInterpExposure, "postISRCCD_uninterpolated")
1771  if result.ossThumb is not None:
1772  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1773  if result.flattenedThumb is not None:
1774  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1775 
1776  return result
1777 
1778  def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1779  """Retrieve a calibration dataset for removing instrument signature.
1780 
1781  Parameters
1782  ----------
1783 
1784  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1785  DataRef of the detector data to find calibration datasets
1786  for.
1787  datasetType : `str`
1788  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1789  dateObs : `str`, optional
1790  Date of the observation. Used to correct butler failures
1791  when using fallback filters.
1792  immediate : `Bool`
1793  If True, disable butler proxies to enable error handling
1794  within this routine.
1795 
1796  Returns
1797  -------
1798  exposure : `lsst.afw.image.Exposure`
1799  Requested calibration frame.
1800 
1801  Raises
1802  ------
1803  RuntimeError
1804  Raised if no matching calibration frame can be found.
1805  """
1806  try:
1807  exp = dataRef.get(datasetType, immediate=immediate)
1808  except Exception as exc1:
1809  if not self.config.fallbackFilterName:
1810  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1811  try:
1812  if self.config.useFallbackDate and dateObs:
1813  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1814  dateObs=dateObs, immediate=immediate)
1815  else:
1816  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1817  except Exception as exc2:
1818  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1819  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1820  self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1821 
1822  if self.config.doAssembleIsrExposures:
1823  exp = self.assembleCcd.assembleCcd(exp)
1824  return exp
1825 
1826  def ensureExposure(self, inputExp, camera, detectorNum):
1827  """Ensure that the data returned by Butler is a fully constructed exposure.
1828 
1829  ISR requires exposure-level image data for historical reasons, so if we did
1830  not recieve that from Butler, construct it from what we have, modifying the
1831  input in place.
1832 
1833  Parameters
1834  ----------
1835  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1836  `lsst.afw.image.ImageF`
1837  The input data structure obtained from Butler.
1838  camera : `lsst.afw.cameraGeom.camera`
1839  The camera associated with the image. Used to find the appropriate
1840  detector.
1841  detectorNum : `int`
1842  The detector this exposure should match.
1843 
1844  Returns
1845  -------
1846  inputExp : `lsst.afw.image.Exposure`
1847  The re-constructed exposure, with appropriate detector parameters.
1848 
1849  Raises
1850  ------
1851  TypeError
1852  Raised if the input data cannot be used to construct an exposure.
1853  """
1854  if isinstance(inputExp, afwImage.DecoratedImageU):
1855  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1856  elif isinstance(inputExp, afwImage.ImageF):
1857  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1858  elif isinstance(inputExp, afwImage.MaskedImageF):
1859  inputExp = afwImage.makeExposure(inputExp)
1860  elif isinstance(inputExp, afwImage.Exposure):
1861  pass
1862  elif inputExp is None:
1863  # Assume this will be caught by the setup if it is a problem.
1864  return inputExp
1865  else:
1866  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1867  (type(inputExp), ))
1868 
1869  if inputExp.getDetector() is None:
1870  inputExp.setDetector(camera[detectorNum])
1871 
1872  return inputExp
1873 
1874  def convertIntToFloat(self, exposure):
1875  """Convert exposure image from uint16 to float.
1876 
1877  If the exposure does not need to be converted, the input is
1878  immediately returned. For exposures that are converted to use
1879  floating point pixels, the variance is set to unity and the
1880  mask to zero.
1881 
1882  Parameters
1883  ----------
1884  exposure : `lsst.afw.image.Exposure`
1885  The raw exposure to be converted.
1886 
1887  Returns
1888  -------
1889  newexposure : `lsst.afw.image.Exposure`
1890  The input ``exposure``, converted to floating point pixels.
1891 
1892  Raises
1893  ------
1894  RuntimeError
1895  Raised if the exposure type cannot be converted to float.
1896 
1897  """
1898  if isinstance(exposure, afwImage.ExposureF):
1899  # Nothing to be done
1900  self.log.debug("Exposure already of type float.")
1901  return exposure
1902  if not hasattr(exposure, "convertF"):
1903  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1904 
1905  newexposure = exposure.convertF()
1906  newexposure.variance[:] = 1
1907  newexposure.mask[:] = 0x0
1908 
1909  return newexposure
1910 
1911  def maskAmplifier(self, ccdExposure, amp, defects):
1912  """Identify bad amplifiers, saturated and suspect pixels.
1913 
1914  Parameters
1915  ----------
1916  ccdExposure : `lsst.afw.image.Exposure`
1917  Input exposure to be masked.
1918  amp : `lsst.afw.table.AmpInfoCatalog`
1919  Catalog of parameters defining the amplifier on this
1920  exposure to mask.
1921  defects : `lsst.ip.isr.Defects`
1922  List of defects. Used to determine if the entire
1923  amplifier is bad.
1924 
1925  Returns
1926  -------
1927  badAmp : `Bool`
1928  If this is true, the entire amplifier area is covered by
1929  defects and unusable.
1930 
1931  """
1932  maskedImage = ccdExposure.getMaskedImage()
1933 
1934  badAmp = False
1935 
1936  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1937  # comparison with current defects definition.
1938  if defects is not None:
1939  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1940 
1941  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1942  # association with pixels in current ccdExposure).
1943  if badAmp:
1944  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1945  afwImage.PARENT)
1946  maskView = dataView.getMask()
1947  maskView |= maskView.getPlaneBitMask("BAD")
1948  del maskView
1949  return badAmp
1950 
1951  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1952  # Saturation and suspect pixels can be masked now, though.
1953  limits = dict()
1954  if self.config.doSaturation and not badAmp:
1955  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1956  if self.config.doSuspect and not badAmp:
1957  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1958  if math.isfinite(self.config.saturation):
1959  limits.update({self.config.saturatedMaskName: self.config.saturation})
1960 
1961  for maskName, maskThreshold in limits.items():
1962  if not math.isnan(maskThreshold):
1963  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1964  isrFunctions.makeThresholdMask(
1965  maskedImage=dataView,
1966  threshold=maskThreshold,
1967  growFootprints=0,
1968  maskName=maskName
1969  )
1970 
1971  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1972  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1973  afwImage.PARENT)
1974  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1975  self.config.suspectMaskName])
1976  if numpy.all(maskView.getArray() & maskVal > 0):
1977  badAmp = True
1978  maskView |= maskView.getPlaneBitMask("BAD")
1979 
1980  return badAmp
1981 
1982  def overscanCorrection(self, ccdExposure, amp):
1983  """Apply overscan correction in place.
1984 
1985  This method does initial pixel rejection of the overscan
1986  region. The overscan can also be optionally segmented to
1987  allow for discontinuous overscan responses to be fit
1988  separately. The actual overscan subtraction is performed by
1989  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1990  which is called here after the amplifier is preprocessed.
1991 
1992  Parameters
1993  ----------
1994  ccdExposure : `lsst.afw.image.Exposure`
1995  Exposure to have overscan correction performed.
1996  amp : `lsst.afw.cameraGeom.Amplifer`
1997  The amplifier to consider while correcting the overscan.
1998 
1999  Returns
2000  -------
2001  overscanResults : `lsst.pipe.base.Struct`
2002  Result struct with components:
2003  - ``imageFit`` : scalar or `lsst.afw.image.Image`
2004  Value or fit subtracted from the amplifier image data.
2005  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
2006  Value or fit subtracted from the overscan image data.
2007  - ``overscanImage`` : `lsst.afw.image.Image`
2008  Image of the overscan region with the overscan
2009  correction applied. This quantity is used to estimate
2010  the amplifier read noise empirically.
2011 
2012  Raises
2013  ------
2014  RuntimeError
2015  Raised if the ``amp`` does not contain raw pixel information.
2016 
2017  See Also
2018  --------
2019  lsst.ip.isr.isrFunctions.overscanCorrection
2020  """
2021  if amp.getRawHorizontalOverscanBBox().isEmpty():
2022  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
2023  return None
2024 
2025  statControl = afwMath.StatisticsControl()
2026  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2027 
2028  # Determine the bounding boxes
2029  dataBBox = amp.getRawDataBBox()
2030  oscanBBox = amp.getRawHorizontalOverscanBBox()
2031  dx0 = 0
2032  dx1 = 0
2033 
2034  prescanBBox = amp.getRawPrescanBBox()
2035  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
2036  dx0 += self.config.overscanNumLeadingColumnsToSkip
2037  dx1 -= self.config.overscanNumTrailingColumnsToSkip
2038  else:
2039  dx0 += self.config.overscanNumTrailingColumnsToSkip
2040  dx1 -= self.config.overscanNumLeadingColumnsToSkip
2041 
2042  # Determine if we need to work on subregions of the amplifier and overscan.
2043  imageBBoxes = []
2044  overscanBBoxes = []
2045 
2046  if ((self.config.overscanBiasJump
2047  and self.config.overscanBiasJumpLocation)
2048  and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2049  and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
2050  self.config.overscanBiasJumpDevices)):
2051  if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
2052  yLower = self.config.overscanBiasJumpLocation
2053  yUpper = dataBBox.getHeight() - yLower
2054  else:
2055  yUpper = self.config.overscanBiasJumpLocation
2056  yLower = dataBBox.getHeight() - yUpper
2057 
2058  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2059  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
2060  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2061  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2062  yLower)))
2063 
2064  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
2065  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
2066  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
2067  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2068  yUpper)))
2069  else:
2070  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2071  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
2072  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2073  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2074  oscanBBox.getHeight())))
2075 
2076  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
2077  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
2078  ampImage = ccdExposure.maskedImage[imageBBox]
2079  overscanImage = ccdExposure.maskedImage[overscanBBox]
2080 
2081  overscanArray = overscanImage.image.array
2082  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2083  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2084  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
2085 
2086  statControl = afwMath.StatisticsControl()
2087  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2088 
2089  overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp)
2090 
2091  # Measure average overscan levels and record them in the metadata.
2092  levelStat = afwMath.MEDIAN
2093  sigmaStat = afwMath.STDEVCLIP
2094 
2095  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2096  self.config.qa.flatness.nIter)
2097  metadata = ccdExposure.getMetadata()
2098  ampNum = amp.getName()
2099  # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
2100  if isinstance(overscanResults.overscanFit, float):
2101  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2102  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2103  else:
2104  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2105  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2106  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2107 
2108  return overscanResults
2109 
2110  def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None):
2111  """Set the variance plane using the gain and read noise
2112 
2113  The read noise is calculated from the ``overscanImage`` if the
2114  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2115  the value from the amplifier data is used.
2116 
2117  Parameters
2118  ----------
2119  ampExposure : `lsst.afw.image.Exposure`
2120  Exposure to process.
2121  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2122  Amplifier detector data.
2123  overscanImage : `lsst.afw.image.MaskedImage`, optional.
2124  Image of overscan, required only for empirical read noise.
2125  ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
2126  PTC dataset containing the gains and read noise.
2127 
2128 
2129  Raises
2130  ------
2131  RuntimeError
2132  Raised if either ``usePtcGains`` of ``usePtcReadNoise``
2133  are ``True``, but ptcDataset is not provided.
2134 
2135  Raised if ```doEmpiricalReadNoise`` is ``True`` but
2136  ``overscanImage`` is ``None``.
2137 
2138  See also
2139  --------
2140  lsst.ip.isr.isrFunctions.updateVariance
2141  """
2142  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2143  if self.config.usePtcGains:
2144  if ptcDataset is None:
2145  raise RuntimeError("No ptcDataset provided to use PTC gains.")
2146  else:
2147  gain = ptcDataset.gain[amp.getName()]
2148  self.log.info("Using gain from Photon Transfer Curve.")
2149  else:
2150  gain = amp.getGain()
2151 
2152  if math.isnan(gain):
2153  gain = 1.0
2154  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2155  elif gain <= 0:
2156  patchedGain = 1.0
2157  self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
2158  amp.getName(), gain, patchedGain)
2159  gain = patchedGain
2160 
2161  if self.config.doEmpiricalReadNoise and overscanImage is None:
2162  raise RuntimeError("Overscan is none for EmpiricalReadNoise.")
2163 
2164  if self.config.doEmpiricalReadNoise and overscanImage is not None:
2165  stats = afwMath.StatisticsControl()
2166  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2167  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2168  self.log.info("Calculated empirical read noise for amp %s: %f.",
2169  amp.getName(), readNoise)
2170  elif self.config.usePtcReadNoise:
2171  if ptcDataset is None:
2172  raise RuntimeError("No ptcDataset provided to use PTC readnoise.")
2173  else:
2174  readNoise = ptcDataset.noise[amp.getName()]
2175  self.log.info("Using read noise from Photon Transfer Curve.")
2176  else:
2177  readNoise = amp.getReadNoise()
2178 
2179  isrFunctions.updateVariance(
2180  maskedImage=ampExposure.getMaskedImage(),
2181  gain=gain,
2182  readNoise=readNoise,
2183  )
2184 
2185  def darkCorrection(self, exposure, darkExposure, invert=False):
2186  """Apply dark correction in place.
2187 
2188  Parameters
2189  ----------
2190  exposure : `lsst.afw.image.Exposure`
2191  Exposure to process.
2192  darkExposure : `lsst.afw.image.Exposure`
2193  Dark exposure of the same size as ``exposure``.
2194  invert : `Bool`, optional
2195  If True, re-add the dark to an already corrected image.
2196 
2197  Raises
2198  ------
2199  RuntimeError
2200  Raised if either ``exposure`` or ``darkExposure`` do not
2201  have their dark time defined.
2202 
2203  See Also
2204  --------
2205  lsst.ip.isr.isrFunctions.darkCorrection
2206  """
2207  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2208  if math.isnan(expScale):
2209  raise RuntimeError("Exposure darktime is NAN.")
2210  if darkExposure.getInfo().getVisitInfo() is not None \
2211  and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2212  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2213  else:
2214  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
2215  # so getDarkTime() does not exist.
2216  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2217  darkScale = 1.0
2218 
2219  isrFunctions.darkCorrection(
2220  maskedImage=exposure.getMaskedImage(),
2221  darkMaskedImage=darkExposure.getMaskedImage(),
2222  expScale=expScale,
2223  darkScale=darkScale,
2224  invert=invert,
2225  trimToFit=self.config.doTrimToMatchCalib
2226  )
2227 
2228  def doLinearize(self, detector):
2229  """Check if linearization is needed for the detector cameraGeom.
2230 
2231  Checks config.doLinearize and the linearity type of the first
2232  amplifier.
2233 
2234  Parameters
2235  ----------
2236  detector : `lsst.afw.cameraGeom.Detector`
2237  Detector to get linearity type from.
2238 
2239  Returns
2240  -------
2241  doLinearize : `Bool`
2242  If True, linearization should be performed.
2243  """
2244  return self.config.doLinearize and \
2245  detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2246 
2247  def flatCorrection(self, exposure, flatExposure, invert=False):
2248  """Apply flat correction in place.
2249 
2250  Parameters
2251  ----------
2252  exposure : `lsst.afw.image.Exposure`
2253  Exposure to process.
2254  flatExposure : `lsst.afw.image.Exposure`
2255  Flat exposure of the same size as ``exposure``.
2256  invert : `Bool`, optional
2257  If True, unflatten an already flattened image.
2258 
2259  See Also
2260  --------
2261  lsst.ip.isr.isrFunctions.flatCorrection
2262  """
2263  isrFunctions.flatCorrection(
2264  maskedImage=exposure.getMaskedImage(),
2265  flatMaskedImage=flatExposure.getMaskedImage(),
2266  scalingType=self.config.flatScalingType,
2267  userScale=self.config.flatUserScale,
2268  invert=invert,
2269  trimToFit=self.config.doTrimToMatchCalib
2270  )
2271 
2272  def saturationDetection(self, exposure, amp):
2273  """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2274 
2275  Parameters
2276  ----------
2277  exposure : `lsst.afw.image.Exposure`
2278  Exposure to process. Only the amplifier DataSec is processed.
2279  amp : `lsst.afw.table.AmpInfoCatalog`
2280  Amplifier detector data.
2281 
2282  See Also
2283  --------
2284  lsst.ip.isr.isrFunctions.makeThresholdMask
2285  """
2286  if not math.isnan(amp.getSaturation()):
2287  maskedImage = exposure.getMaskedImage()
2288  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2289  isrFunctions.makeThresholdMask(
2290  maskedImage=dataView,
2291  threshold=amp.getSaturation(),
2292  growFootprints=0,
2293  maskName=self.config.saturatedMaskName,
2294  )
2295 
2296  def saturationInterpolation(self, exposure):
2297  """Interpolate over saturated pixels, in place.
2298 
2299  This method should be called after `saturationDetection`, to
2300  ensure that the saturated pixels have been identified in the
2301  SAT mask. It should also be called after `assembleCcd`, since
2302  saturated regions may cross amplifier boundaries.
2303 
2304  Parameters
2305  ----------
2306  exposure : `lsst.afw.image.Exposure`
2307  Exposure to process.
2308 
2309  See Also
2310  --------
2311  lsst.ip.isr.isrTask.saturationDetection
2312  lsst.ip.isr.isrFunctions.interpolateFromMask
2313  """
2314  isrFunctions.interpolateFromMask(
2315  maskedImage=exposure.getMaskedImage(),
2316  fwhm=self.config.fwhm,
2317  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2318  maskNameList=list(self.config.saturatedMaskName),
2319  )
2320 
2321  def suspectDetection(self, exposure, amp):
2322  """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2323 
2324  Parameters
2325  ----------
2326  exposure : `lsst.afw.image.Exposure`
2327  Exposure to process. Only the amplifier DataSec is processed.
2328  amp : `lsst.afw.table.AmpInfoCatalog`
2329  Amplifier detector data.
2330 
2331  See Also
2332  --------
2333  lsst.ip.isr.isrFunctions.makeThresholdMask
2334 
2335  Notes
2336  -----
2337  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2338  This is intended to indicate pixels that may be affected by unknown systematics;
2339  for example if non-linearity corrections above a certain level are unstable
2340  then that would be a useful value for suspectLevel. A value of `nan` indicates
2341  that no such level exists and no pixels are to be masked as suspicious.
2342  """
2343  suspectLevel = amp.getSuspectLevel()
2344  if math.isnan(suspectLevel):
2345  return
2346 
2347  maskedImage = exposure.getMaskedImage()
2348  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2349  isrFunctions.makeThresholdMask(
2350  maskedImage=dataView,
2351  threshold=suspectLevel,
2352  growFootprints=0,
2353  maskName=self.config.suspectMaskName,
2354  )
2355 
2356  def maskDefect(self, exposure, defectBaseList):
2357  """Mask defects using mask plane "BAD", in place.
2358 
2359  Parameters
2360  ----------
2361  exposure : `lsst.afw.image.Exposure`
2362  Exposure to process.
2363  defectBaseList : `lsst.ip.isr.Defects` or `list` of
2364  `lsst.afw.image.DefectBase`.
2365  List of defects to mask.
2366 
2367  Notes
2368  -----
2369  Call this after CCD assembly, since defects may cross amplifier boundaries.
2370  """
2371  maskedImage = exposure.getMaskedImage()
2372  if not isinstance(defectBaseList, Defects):
2373  # Promotes DefectBase to Defect
2374  defectList = Defects(defectBaseList)
2375  else:
2376  defectList = defectBaseList
2377  defectList.maskPixels(maskedImage, maskName="BAD")
2378 
2379  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2380  """Mask edge pixels with applicable mask plane.
2381 
2382  Parameters
2383  ----------
2384  exposure : `lsst.afw.image.Exposure`
2385  Exposure to process.
2386  numEdgePixels : `int`, optional
2387  Number of edge pixels to mask.
2388  maskPlane : `str`, optional
2389  Mask plane name to use.
2390  level : `str`, optional
2391  Level at which to mask edges.
2392  """
2393  maskedImage = exposure.getMaskedImage()
2394  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2395 
2396  if numEdgePixels > 0:
2397  if level == 'DETECTOR':
2398  boxes = [maskedImage.getBBox()]
2399  elif level == 'AMP':
2400  boxes = [amp.getBBox() for amp in exposure.getDetector()]
2401 
2402  for box in boxes:
2403  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2404  subImage = maskedImage[box]
2405  box.grow(-numEdgePixels)
2406  # Mask pixels outside box
2407  SourceDetectionTask.setEdgeBits(
2408  subImage,
2409  box,
2410  maskBitMask)
2411 
2412  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2413  """Mask and interpolate defects using mask plane "BAD", in place.
2414 
2415  Parameters
2416  ----------
2417  exposure : `lsst.afw.image.Exposure`
2418  Exposure to process.
2419  defectBaseList : `lsst.ip.isr.Defects` or `list` of
2420  `lsst.afw.image.DefectBase`.
2421  List of defects to mask and interpolate.
2422 
2423  See Also
2424  --------
2425  lsst.ip.isr.isrTask.maskDefect
2426  """
2427  self.maskDefectmaskDefect(exposure, defectBaseList)
2428  self.maskEdgesmaskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2429  maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2430  isrFunctions.interpolateFromMask(
2431  maskedImage=exposure.getMaskedImage(),
2432  fwhm=self.config.fwhm,
2433  growSaturatedFootprints=0,
2434  maskNameList=["BAD"],
2435  )
2436 
2437  def maskNan(self, exposure):
2438  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2439 
2440  Parameters
2441  ----------
2442  exposure : `lsst.afw.image.Exposure`
2443  Exposure to process.
2444 
2445  Notes
2446  -----
2447  We mask over all non-finite values (NaN, inf), including those
2448  that are masked with other bits (because those may or may not be
2449  interpolated over later, and we want to remove all NaN/infs).
2450  Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2451  preserve the historical name.
2452  """
2453  maskedImage = exposure.getMaskedImage()
2454 
2455  # Find and mask NaNs
2456  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2457  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2458  numNans = maskNans(maskedImage, maskVal)
2459  self.metadata.set("NUMNANS", numNans)
2460  if numNans > 0:
2461  self.log.warn("There were %d unmasked NaNs.", numNans)
2462 
2463  def maskAndInterpolateNan(self, exposure):
2464  """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2465  in place.
2466 
2467  Parameters
2468  ----------
2469  exposure : `lsst.afw.image.Exposure`
2470  Exposure to process.
2471 
2472  See Also
2473  --------
2474  lsst.ip.isr.isrTask.maskNan
2475  """
2476  self.maskNanmaskNan(exposure)
2477  isrFunctions.interpolateFromMask(
2478  maskedImage=exposure.getMaskedImage(),
2479  fwhm=self.config.fwhm,
2480  growSaturatedFootprints=0,
2481  maskNameList=["UNMASKEDNAN"],
2482  )
2483 
2484  def measureBackground(self, exposure, IsrQaConfig=None):
2485  """Measure the image background in subgrids, for quality control purposes.
2486 
2487  Parameters
2488  ----------
2489  exposure : `lsst.afw.image.Exposure`
2490  Exposure to process.
2491  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2492  Configuration object containing parameters on which background
2493  statistics and subgrids to use.
2494  """
2495  if IsrQaConfig is not None:
2496  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2497  IsrQaConfig.flatness.nIter)
2498  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2499  statsControl.setAndMask(maskVal)
2500  maskedImage = exposure.getMaskedImage()
2501  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2502  skyLevel = stats.getValue(afwMath.MEDIAN)
2503  skySigma = stats.getValue(afwMath.STDEVCLIP)
2504  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2505  metadata = exposure.getMetadata()
2506  metadata.set('SKYLEVEL', skyLevel)
2507  metadata.set('SKYSIGMA', skySigma)
2508 
2509  # calcluating flatlevel over the subgrids
2510  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2511  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2512  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2513  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2514  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2515  skyLevels = numpy.zeros((nX, nY))
2516 
2517  for j in range(nY):
2518  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2519  for i in range(nX):
2520  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2521 
2522  xLLC = xc - meshXHalf
2523  yLLC = yc - meshYHalf
2524  xURC = xc + meshXHalf - 1
2525  yURC = yc + meshYHalf - 1
2526 
2527  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2528  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2529 
2530  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2531 
2532  good = numpy.where(numpy.isfinite(skyLevels))
2533  skyMedian = numpy.median(skyLevels[good])
2534  flatness = (skyLevels[good] - skyMedian) / skyMedian
2535  flatness_rms = numpy.std(flatness)
2536  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2537 
2538  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2539  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2540  nX, nY, flatness_pp, flatness_rms)
2541 
2542  metadata.set('FLATNESS_PP', float(flatness_pp))
2543  metadata.set('FLATNESS_RMS', float(flatness_rms))
2544  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2545  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2546  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2547 
2548  def roughZeroPoint(self, exposure):
2549  """Set an approximate magnitude zero point for the exposure.
2550 
2551  Parameters
2552  ----------
2553  exposure : `lsst.afw.image.Exposure`
2554  Exposure to process.
2555  """
2556  filterLabel = exposure.getFilterLabel()
2557  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2558 
2559  if physicalFilter in self.config.fluxMag0T1:
2560  fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2561  else:
2562  self.log.warn("No rough magnitude zero point defined for filter {}.".format(physicalFilter))
2563  fluxMag0 = self.config.defaultFluxMag0T1
2564 
2565  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2566  if not expTime > 0: # handle NaN as well as <= 0
2567  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2568  return
2569 
2570  self.log.info("Setting rough magnitude zero point for filter {}: {}".
2571  format(physicalFilter, 2.5*math.log10(fluxMag0*expTime)))
2572  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2573 
2574  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2575  """Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2576 
2577  Parameters
2578  ----------
2579  ccdExposure : `lsst.afw.image.Exposure`
2580  Exposure to process.
2581  fpPolygon : `lsst.afw.geom.Polygon`
2582  Polygon in focal plane coordinates.
2583  """
2584  # Get ccd corners in focal plane coordinates
2585  ccd = ccdExposure.getDetector()
2586  fpCorners = ccd.getCorners(FOCAL_PLANE)
2587  ccdPolygon = Polygon(fpCorners)
2588 
2589  # Get intersection of ccd corners with fpPolygon
2590  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2591 
2592  # Transform back to pixel positions and build new polygon
2593  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2594  validPolygon = Polygon(ccdPoints)
2595  ccdExposure.getInfo().setValidPolygon(validPolygon)
2596 
2597  @contextmanager
2598  def flatContext(self, exp, flat, dark=None):
2599  """Context manager that applies and removes flats and darks,
2600  if the task is configured to apply them.
2601 
2602  Parameters
2603  ----------
2604  exp : `lsst.afw.image.Exposure`
2605  Exposure to process.
2606  flat : `lsst.afw.image.Exposure`
2607  Flat exposure the same size as ``exp``.
2608  dark : `lsst.afw.image.Exposure`, optional
2609  Dark exposure the same size as ``exp``.
2610 
2611  Yields
2612  ------
2613  exp : `lsst.afw.image.Exposure`
2614  The flat and dark corrected exposure.
2615  """
2616  if self.config.doDark and dark is not None:
2617  self.darkCorrectiondarkCorrection(exp, dark)
2618  if self.config.doFlat:
2619  self.flatCorrectionflatCorrection(exp, flat)
2620  try:
2621  yield exp
2622  finally:
2623  if self.config.doFlat:
2624  self.flatCorrectionflatCorrection(exp, flat, invert=True)
2625  if self.config.doDark and dark is not None:
2626  self.darkCorrectiondarkCorrection(exp, dark, invert=True)
2627 
2628  def debugView(self, exposure, stepname):
2629  """Utility function to examine ISR exposure at different stages.
2630 
2631  Parameters
2632  ----------
2633  exposure : `lsst.afw.image.Exposure`
2634  Exposure to view.
2635  stepname : `str`
2636  State of processing to view.
2637  """
2638  frame = getDebugFrame(self._display, stepname)
2639  if frame:
2640  display = getDisplay(frame)
2641  display.scale('asinh', 'zscale')
2642  display.mtv(exposure)
2643  prompt = "Press Enter to continue [c]... "
2644  while True:
2645  ans = input(prompt).lower()
2646  if ans in ("", "c",):
2647  break
2648 
2649 
2650 class FakeAmp(object):
2651  """A Detector-like object that supports returning gain and saturation level
2652 
2653  This is used when the input exposure does not have a detector.
2654 
2655  Parameters
2656  ----------
2657  exposure : `lsst.afw.image.Exposure`
2658  Exposure to generate a fake amplifier for.
2659  config : `lsst.ip.isr.isrTaskConfig`
2660  Configuration to apply to the fake amplifier.
2661  """
2662 
2663  def __init__(self, exposure, config):
2664  self._bbox_bbox = exposure.getBBox(afwImage.LOCAL)
2665  self._RawHorizontalOverscanBBox_RawHorizontalOverscanBBox = lsst.geom.Box2I()
2666  self._gain_gain = config.gain
2667  self._readNoise_readNoise = config.readNoise
2668  self._saturation_saturation = config.saturation
2669 
2670  def getBBox(self):
2671  return self._bbox_bbox
2672 
2673  def getRawBBox(self):
2674  return self._bbox_bbox
2675 
2677  return self._RawHorizontalOverscanBBox_RawHorizontalOverscanBBox
2678 
2679  def getGain(self):
2680  return self._gain_gain
2681 
2682  def getReadNoise(self):
2683  return self._readNoise_readNoise
2684 
2685  def getSaturation(self):
2686  return self._saturation_saturation
2687 
2688  def getSuspectLevel(self):
2689  return float("NaN")
2690 
2691 
2692 class RunIsrConfig(pexConfig.Config):
2693  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2694 
2695 
2696 class RunIsrTask(pipeBase.CmdLineTask):
2697  """Task to wrap the default IsrTask to allow it to be retargeted.
2698 
2699  The standard IsrTask can be called directly from a command line
2700  program, but doing so removes the ability of the task to be
2701  retargeted. As most cameras override some set of the IsrTask
2702  methods, this would remove those data-specific methods in the
2703  output post-ISR images. This wrapping class fixes the issue,
2704  allowing identical post-ISR images to be generated by both the
2705  processCcd and isrTask code.
2706  """
2707  ConfigClass = RunIsrConfig
2708  _DefaultName = "runIsr"
2709 
2710  def __init__(self, *args, **kwargs):
2711  super().__init__(*args, **kwargs)
2712  self.makeSubtask("isr")
2713 
2714  def runDataRef(self, dataRef):
2715  """
2716  Parameters
2717  ----------
2718  dataRef : `lsst.daf.persistence.ButlerDataRef`
2719  data reference of the detector data to be processed
2720 
2721  Returns
2722  -------
2723  result : `pipeBase.Struct`
2724  Result struct with component:
2725 
2726  - exposure : `lsst.afw.image.Exposure`
2727  Post-ISR processed exposure.
2728  """
2729  return self.isr.runDataRef(dataRef)
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2676
def __init__(self, exposure, config)
Definition: isrTask.py:2663
def __init__(self, *config=None)
Definition: isrTask.py:272
def flatCorrection(self, exposure, flatExposure, invert=False)
Definition: isrTask.py:2247
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2463
def saturationInterpolation(self, exposure)
Definition: isrTask.py:2296
def runDataRef(self, sensorRef)
Definition: isrTask.py:1727
def maskNan(self, exposure)
Definition: isrTask.py:2437
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1911
def debugView(self, exposure, stepname)
Definition: isrTask.py:2628
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Definition: isrTask.py:1778
def saturationDetection(self, exposure, amp)
Definition: isrTask.py:2272
def maskDefect(self, exposure, defectBaseList)
Definition: isrTask.py:2356
def __init__(self, **kwargs)
Definition: isrTask.py:948
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:958
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
Definition: isrTask.py:2379
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1982
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2484
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2548
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2412
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Definition: isrTask.py:2574
def readIsrData(self, dataRef, rawExposure)
Definition: isrTask.py:1055
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1826
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Definition: isrTask.py:1237
def doLinearize(self, detector)
Definition: isrTask.py:2228
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2598
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1874
def suspectDetection(self, exposure, amp)
Definition: isrTask.py:2321
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
Definition: isrTask.py:2110
def darkCorrection(self, exposure, darkExposure, invert=False)
Definition: isrTask.py:2185
def __init__(self, *args, **kwargs)
Definition: isrTask.py:2710
def runDataRef(self, dataRef)
Definition: isrTask.py:2714
def checkFilter(exposure, filterList, log)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
Definition: isrTask.py:62
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35