lsst.ip.isr  21.0.0-19-g6ed576e+e58e3478d4
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
30 import lsst.pipe.base.connectionTypes as cT
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36  ReadoutCorner)
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 
43 from . import isrFunctions
44 from . import isrQa
45 from . import linearize
46 from .defects import Defects
47 
48 from .assembleCcdTask import AssembleCcdTask
49 from .crosstalk import CrosstalkTask, CrosstalkCalib
50 from .fringe import FringeTask
51 from .isr import maskNans
52 from .masking import MaskingTask
53 from .overscan import OverscanCorrectionTask
54 from .straylight import StrayLightTask
55 from .vignette import VignetteTask
56 from lsst.daf.butler import DimensionGraph
57 
58 
59 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
60 
61 
62 def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections):
63  """Lookup function to identify crosstalkSource entries.
64 
65  This should return an empty list under most circumstances. Only
66  when inter-chip crosstalk has been identified should this be
67  populated.
68 
69  This will be unused until DM-25348 resolves the quantum graph
70  generation issue.
71 
72  Parameters
73  ----------
74  datasetType : `str`
75  Dataset to lookup.
76  registry : `lsst.daf.butler.Registry`
77  Butler registry to query.
78  quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79  Data id to transform to identify crosstalkSources. The
80  ``detector`` entry will be stripped.
81  collections : `lsst.daf.butler.CollectionSearch`
82  Collections to search through.
83 
84  Returns
85  -------
86  results : `list` [`lsst.daf.butler.DatasetRef`]
87  List of datasets that match the query that will be used as
88  crosstalkSources.
89  """
90  newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"]))
91  results = list(registry.queryDatasets(datasetType,
92  collections=collections,
93  dataId=newDataId,
94  findFirst=True,
95  ).expanded())
96  return results
97 
98 
99 class IsrTaskConnections(pipeBase.PipelineTaskConnections,
100  dimensions={"instrument", "exposure", "detector"},
101  defaultTemplates={}):
102  ccdExposure = cT.Input(
103  name="raw",
104  doc="Input exposure to process.",
105  storageClass="Exposure",
106  dimensions=["instrument", "exposure", "detector"],
107  )
108  camera = cT.PrerequisiteInput(
109  name="camera",
110  storageClass="Camera",
111  doc="Input camera to construct complete exposures.",
112  dimensions=["instrument"],
113  isCalibration=True,
114  )
115 
116  crosstalk = cT.PrerequisiteInput(
117  name="crosstalk",
118  doc="Input crosstalk object",
119  storageClass="CrosstalkCalib",
120  dimensions=["instrument", "detector"],
121  isCalibration=True,
122  )
123  # TODO: DM-25348. This does not work yet to correctly load
124  # possible crosstalk sources.
125  crosstalkSources = cT.PrerequisiteInput(
126  name="isrOverscanCorrected",
127  doc="Overscan corrected input images.",
128  storageClass="Exposure",
129  dimensions=["instrument", "exposure", "detector"],
130  deferLoad=True,
131  multiple=True,
132  lookupFunction=crosstalkSourceLookup,
133  )
134  bias = cT.PrerequisiteInput(
135  name="bias",
136  doc="Input bias calibration.",
137  storageClass="ExposureF",
138  dimensions=["instrument", "detector"],
139  isCalibration=True,
140  )
141  dark = cT.PrerequisiteInput(
142  name='dark',
143  doc="Input dark calibration.",
144  storageClass="ExposureF",
145  dimensions=["instrument", "detector"],
146  isCalibration=True,
147  )
148  flat = cT.PrerequisiteInput(
149  name="flat",
150  doc="Input flat calibration.",
151  storageClass="ExposureF",
152  dimensions=["instrument", "physical_filter", "detector"],
153  isCalibration=True,
154  )
155  ptc = cT.PrerequisiteInput(
156  name="ptc",
157  doc="Input Photon Transfer Curve dataset",
158  storageClass="PhotonTransferCurveDataset",
159  dimensions=["instrument", "detector"],
160  isCalibration=True,
161  )
162  fringes = cT.PrerequisiteInput(
163  name="fringe",
164  doc="Input fringe calibration.",
165  storageClass="ExposureF",
166  dimensions=["instrument", "physical_filter", "detector"],
167  isCalibration=True,
168  )
169  strayLightData = cT.PrerequisiteInput(
170  name='yBackground',
171  doc="Input stray light calibration.",
172  storageClass="StrayLightData",
173  dimensions=["instrument", "physical_filter", "detector"],
174  isCalibration=True,
175  )
176  bfKernel = cT.PrerequisiteInput(
177  name='bfKernel',
178  doc="Input brighter-fatter kernel.",
179  storageClass="NumpyArray",
180  dimensions=["instrument"],
181  isCalibration=True,
182  )
183  newBFKernel = cT.PrerequisiteInput(
184  name='brighterFatterKernel',
185  doc="Newer complete kernel + gain solutions.",
186  storageClass="BrighterFatterKernel",
187  dimensions=["instrument", "detector"],
188  isCalibration=True,
189  )
190  defects = cT.PrerequisiteInput(
191  name='defects',
192  doc="Input defect tables.",
193  storageClass="Defects",
194  dimensions=["instrument", "detector"],
195  isCalibration=True,
196  )
197  linearizer = cT.PrerequisiteInput(
198  name='linearizer',
199  storageClass="Linearizer",
200  doc="Linearity correction calibration.",
201  dimensions=["instrument", "detector"],
202  isCalibration=True,
203  )
204  opticsTransmission = cT.PrerequisiteInput(
205  name="transmission_optics",
206  storageClass="TransmissionCurve",
207  doc="Transmission curve due to the optics.",
208  dimensions=["instrument"],
209  isCalibration=True,
210  )
211  filterTransmission = cT.PrerequisiteInput(
212  name="transmission_filter",
213  storageClass="TransmissionCurve",
214  doc="Transmission curve due to the filter.",
215  dimensions=["instrument", "physical_filter"],
216  isCalibration=True,
217  )
218  sensorTransmission = cT.PrerequisiteInput(
219  name="transmission_sensor",
220  storageClass="TransmissionCurve",
221  doc="Transmission curve due to the sensor.",
222  dimensions=["instrument", "detector"],
223  isCalibration=True,
224  )
225  atmosphereTransmission = cT.PrerequisiteInput(
226  name="transmission_atmosphere",
227  storageClass="TransmissionCurve",
228  doc="Transmission curve due to the atmosphere.",
229  dimensions=["instrument"],
230  isCalibration=True,
231  )
232  illumMaskedImage = cT.PrerequisiteInput(
233  name="illum",
234  doc="Input illumination correction.",
235  storageClass="MaskedImageF",
236  dimensions=["instrument", "physical_filter", "detector"],
237  isCalibration=True,
238  )
239 
240  outputExposure = cT.Output(
241  name='postISRCCD',
242  doc="Output ISR processed exposure.",
243  storageClass="Exposure",
244  dimensions=["instrument", "exposure", "detector"],
245  )
246  preInterpExposure = cT.Output(
247  name='preInterpISRCCD',
248  doc="Output ISR processed exposure, with pixels left uninterpolated.",
249  storageClass="ExposureF",
250  dimensions=["instrument", "exposure", "detector"],
251  )
252  outputOssThumbnail = cT.Output(
253  name="OssThumb",
254  doc="Output Overscan-subtracted thumbnail image.",
255  storageClass="Thumbnail",
256  dimensions=["instrument", "exposure", "detector"],
257  )
258  outputFlattenedThumbnail = cT.Output(
259  name="FlattenedThumb",
260  doc="Output flat-corrected thumbnail image.",
261  storageClass="Thumbnail",
262  dimensions=["instrument", "exposure", "detector"],
263  )
264 
265  def __init__(self, *, config=None):
266  super().__init__(config=config)
267 
268  if config.doBias is not True:
269  self.prerequisiteInputs.discard("bias")
270  if config.doLinearize is not True:
271  self.prerequisiteInputs.discard("linearizer")
272  if config.doCrosstalk is not True:
273  self.inputs.discard("crosstalkSources")
274  self.prerequisiteInputs.discard("crosstalk")
275  if config.doBrighterFatter is not True:
276  self.prerequisiteInputs.discard("bfKernel")
277  self.prerequisiteInputs.discard("newBFKernel")
278  if config.doDefect is not True:
279  self.prerequisiteInputs.discard("defects")
280  if config.doDark is not True:
281  self.prerequisiteInputs.discard("dark")
282  if config.doFlat is not True:
283  self.prerequisiteInputs.discard("flat")
284  if config.usePtcGains is not True and config.usePtcReadNoise is not True:
285  self.prerequisiteInputs.discard("ptc")
286  if config.doAttachTransmissionCurve is not True:
287  self.prerequisiteInputs.discard("opticsTransmission")
288  self.prerequisiteInputs.discard("filterTransmission")
289  self.prerequisiteInputs.discard("sensorTransmission")
290  self.prerequisiteInputs.discard("atmosphereTransmission")
291  if config.doUseOpticsTransmission is not True:
292  self.prerequisiteInputs.discard("opticsTransmission")
293  if config.doUseFilterTransmission is not True:
294  self.prerequisiteInputs.discard("filterTransmission")
295  if config.doUseSensorTransmission is not True:
296  self.prerequisiteInputs.discard("sensorTransmission")
297  if config.doUseAtmosphereTransmission is not True:
298  self.prerequisiteInputs.discard("atmosphereTransmission")
299  if config.doIlluminationCorrection is not True:
300  self.prerequisiteInputs.discard("illumMaskedImage")
301 
302  if config.doWrite is not True:
303  self.outputs.discard("outputExposure")
304  self.outputs.discard("preInterpExposure")
305  self.outputs.discard("outputFlattenedThumbnail")
306  self.outputs.discard("outputOssThumbnail")
307  if config.doSaveInterpPixels is not True:
308  self.outputs.discard("preInterpExposure")
309  if config.qa.doThumbnailOss is not True:
310  self.outputs.discard("outputOssThumbnail")
311  if config.qa.doThumbnailFlattened is not True:
312  self.outputs.discard("outputFlattenedThumbnail")
313 
314 
315 class IsrTaskConfig(pipeBase.PipelineTaskConfig,
316  pipelineConnections=IsrTaskConnections):
317  """Configuration parameters for IsrTask.
318 
319  Items are grouped in the order in which they are executed by the task.
320  """
321  datasetType = pexConfig.Field(
322  dtype=str,
323  doc="Dataset type for input data; users will typically leave this alone, "
324  "but camera-specific ISR tasks will override it",
325  default="raw",
326  )
327 
328  fallbackFilterName = pexConfig.Field(
329  dtype=str,
330  doc="Fallback default filter name for calibrations.",
331  optional=True
332  )
333  useFallbackDate = pexConfig.Field(
334  dtype=bool,
335  doc="Pass observation date when using fallback filter.",
336  default=False,
337  )
338  expectWcs = pexConfig.Field(
339  dtype=bool,
340  default=True,
341  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
342  )
343  fwhm = pexConfig.Field(
344  dtype=float,
345  doc="FWHM of PSF in arcseconds.",
346  default=1.0,
347  )
348  qa = pexConfig.ConfigField(
349  dtype=isrQa.IsrQaConfig,
350  doc="QA related configuration options.",
351  )
352 
353  # Image conversion configuration
354  doConvertIntToFloat = pexConfig.Field(
355  dtype=bool,
356  doc="Convert integer raw images to floating point values?",
357  default=True,
358  )
359 
360  # Saturated pixel handling.
361  doSaturation = pexConfig.Field(
362  dtype=bool,
363  doc="Mask saturated pixels? NB: this is totally independent of the"
364  " interpolation option - this is ONLY setting the bits in the mask."
365  " To have them interpolated make sure doSaturationInterpolation=True",
366  default=True,
367  )
368  saturatedMaskName = pexConfig.Field(
369  dtype=str,
370  doc="Name of mask plane to use in saturation detection and interpolation",
371  default="SAT",
372  )
373  saturation = pexConfig.Field(
374  dtype=float,
375  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
376  default=float("NaN"),
377  )
378  growSaturationFootprintSize = pexConfig.Field(
379  dtype=int,
380  doc="Number of pixels by which to grow the saturation footprints",
381  default=1,
382  )
383 
384  # Suspect pixel handling.
385  doSuspect = pexConfig.Field(
386  dtype=bool,
387  doc="Mask suspect pixels?",
388  default=False,
389  )
390  suspectMaskName = pexConfig.Field(
391  dtype=str,
392  doc="Name of mask plane to use for suspect pixels",
393  default="SUSPECT",
394  )
395  numEdgeSuspect = pexConfig.Field(
396  dtype=int,
397  doc="Number of edge pixels to be flagged as untrustworthy.",
398  default=0,
399  )
400  edgeMaskLevel = pexConfig.ChoiceField(
401  dtype=str,
402  doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
403  default="DETECTOR",
404  allowed={
405  'DETECTOR': 'Mask only the edges of the full detector.',
406  'AMP': 'Mask edges of each amplifier.',
407  },
408  )
409 
410  # Initial masking options.
411  doSetBadRegions = pexConfig.Field(
412  dtype=bool,
413  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
414  default=True,
415  )
416  badStatistic = pexConfig.ChoiceField(
417  dtype=str,
418  doc="How to estimate the average value for BAD regions.",
419  default='MEANCLIP',
420  allowed={
421  "MEANCLIP": "Correct using the (clipped) mean of good data",
422  "MEDIAN": "Correct using the median of the good data",
423  },
424  )
425 
426  # Overscan subtraction configuration.
427  doOverscan = pexConfig.Field(
428  dtype=bool,
429  doc="Do overscan subtraction?",
430  default=True,
431  )
432  overscan = pexConfig.ConfigurableField(
433  target=OverscanCorrectionTask,
434  doc="Overscan subtraction task for image segments.",
435  )
436 
437  overscanFitType = pexConfig.ChoiceField(
438  dtype=str,
439  doc="The method for fitting the overscan bias level.",
440  default='MEDIAN',
441  allowed={
442  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
443  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
444  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
445  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
446  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
447  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
448  "MEAN": "Correct using the mean of the overscan region",
449  "MEANCLIP": "Correct using a clipped mean of the overscan region",
450  "MEDIAN": "Correct using the median of the overscan region",
451  "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
452  },
453  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
454  " This option will no longer be used, and will be removed after v20.")
455  )
456  overscanOrder = pexConfig.Field(
457  dtype=int,
458  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, "
459  "or number of spline knots if overscan fit type is a spline."),
460  default=1,
461  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
462  " This option will no longer be used, and will be removed after v20.")
463  )
464  overscanNumSigmaClip = pexConfig.Field(
465  dtype=float,
466  doc="Rejection threshold (sigma) for collapsing overscan before fit",
467  default=3.0,
468  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
469  " This option will no longer be used, and will be removed after v20.")
470  )
471  overscanIsInt = pexConfig.Field(
472  dtype=bool,
473  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
474  " and overscan.FitType=MEDIAN_PER_ROW.",
475  default=True,
476  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
477  " This option will no longer be used, and will be removed after v20.")
478  )
479  # These options do not get deprecated, as they define how we slice up the image data.
480  overscanNumLeadingColumnsToSkip = pexConfig.Field(
481  dtype=int,
482  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
483  default=0,
484  )
485  overscanNumTrailingColumnsToSkip = pexConfig.Field(
486  dtype=int,
487  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
488  default=0,
489  )
490  overscanMaxDev = pexConfig.Field(
491  dtype=float,
492  doc="Maximum deviation from the median for overscan",
493  default=1000.0, check=lambda x: x > 0
494  )
495  overscanBiasJump = pexConfig.Field(
496  dtype=bool,
497  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
498  default=False,
499  )
500  overscanBiasJumpKeyword = pexConfig.Field(
501  dtype=str,
502  doc="Header keyword containing information about devices.",
503  default="NO_SUCH_KEY",
504  )
505  overscanBiasJumpDevices = pexConfig.ListField(
506  dtype=str,
507  doc="List of devices that need piecewise overscan correction.",
508  default=(),
509  )
510  overscanBiasJumpLocation = pexConfig.Field(
511  dtype=int,
512  doc="Location of bias jump along y-axis.",
513  default=0,
514  )
515 
516  # Amplifier to CCD assembly configuration
517  doAssembleCcd = pexConfig.Field(
518  dtype=bool,
519  default=True,
520  doc="Assemble amp-level exposures into a ccd-level exposure?"
521  )
522  assembleCcd = pexConfig.ConfigurableField(
523  target=AssembleCcdTask,
524  doc="CCD assembly task",
525  )
526 
527  # General calibration configuration.
528  doAssembleIsrExposures = pexConfig.Field(
529  dtype=bool,
530  default=False,
531  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
532  )
533  doTrimToMatchCalib = pexConfig.Field(
534  dtype=bool,
535  default=False,
536  doc="Trim raw data to match calibration bounding boxes?"
537  )
538 
539  # Bias subtraction.
540  doBias = pexConfig.Field(
541  dtype=bool,
542  doc="Apply bias frame correction?",
543  default=True,
544  )
545  biasDataProductName = pexConfig.Field(
546  dtype=str,
547  doc="Name of the bias data product",
548  default="bias",
549  )
550  doBiasBeforeOverscan = pexConfig.Field(
551  dtype=bool,
552  doc="Reverse order of overscan and bias correction.",
553  default=False
554  )
555 
556  # Variance construction
557  doVariance = pexConfig.Field(
558  dtype=bool,
559  doc="Calculate variance?",
560  default=True
561  )
562  gain = pexConfig.Field(
563  dtype=float,
564  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
565  default=float("NaN"),
566  )
567  readNoise = pexConfig.Field(
568  dtype=float,
569  doc="The read noise to use if no Detector is present in the Exposure",
570  default=0.0,
571  )
572  doEmpiricalReadNoise = pexConfig.Field(
573  dtype=bool,
574  default=False,
575  doc="Calculate empirical read noise instead of value from AmpInfo data?"
576  )
577  usePtcReadNoise = pexConfig.Field(
578  dtype=bool,
579  default=False,
580  doc="Use readnoise values from the Photon Transfer Curve?"
581  )
582  # Linearization.
583  doLinearize = pexConfig.Field(
584  dtype=bool,
585  doc="Correct for nonlinearity of the detector's response?",
586  default=True,
587  )
588 
589  # Crosstalk.
590  doCrosstalk = pexConfig.Field(
591  dtype=bool,
592  doc="Apply intra-CCD crosstalk correction?",
593  default=False,
594  )
595  doCrosstalkBeforeAssemble = pexConfig.Field(
596  dtype=bool,
597  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
598  default=False,
599  )
600  crosstalk = pexConfig.ConfigurableField(
601  target=CrosstalkTask,
602  doc="Intra-CCD crosstalk correction",
603  )
604 
605  # Masking options.
606  doDefect = pexConfig.Field(
607  dtype=bool,
608  doc="Apply correction for CCD defects, e.g. hot pixels?",
609  default=True,
610  )
611  doNanMasking = pexConfig.Field(
612  dtype=bool,
613  doc="Mask non-finite (NAN, inf) pixels?",
614  default=True,
615  )
616  doWidenSaturationTrails = pexConfig.Field(
617  dtype=bool,
618  doc="Widen bleed trails based on their width?",
619  default=True
620  )
621 
622  # Brighter-Fatter correction.
623  doBrighterFatter = pexConfig.Field(
624  dtype=bool,
625  default=False,
626  doc="Apply the brighter-fatter correction?"
627  )
628  brighterFatterLevel = pexConfig.ChoiceField(
629  dtype=str,
630  default="DETECTOR",
631  doc="The level at which to correct for brighter-fatter.",
632  allowed={
633  "AMP": "Every amplifier treated separately.",
634  "DETECTOR": "One kernel per detector",
635  }
636  )
637  brighterFatterMaxIter = pexConfig.Field(
638  dtype=int,
639  default=10,
640  doc="Maximum number of iterations for the brighter-fatter correction"
641  )
642  brighterFatterThreshold = pexConfig.Field(
643  dtype=float,
644  default=1000,
645  doc="Threshold used to stop iterating the brighter-fatter correction. It is the "
646  "absolute value of the difference between the current corrected image and the one "
647  "from the previous iteration summed over all the pixels."
648  )
649  brighterFatterApplyGain = pexConfig.Field(
650  dtype=bool,
651  default=True,
652  doc="Should the gain be applied when applying the brighter-fatter correction?"
653  )
654  brighterFatterMaskListToInterpolate = pexConfig.ListField(
655  dtype=str,
656  doc="List of mask planes that should be interpolated over when applying the brighter-fatter "
657  "correction.",
658  default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"],
659  )
660  brighterFatterMaskGrowSize = pexConfig.Field(
661  dtype=int,
662  default=0,
663  doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
664  "when brighter-fatter correction is applied."
665  )
666 
667  # Dark subtraction.
668  doDark = pexConfig.Field(
669  dtype=bool,
670  doc="Apply dark frame correction?",
671  default=True,
672  )
673  darkDataProductName = pexConfig.Field(
674  dtype=str,
675  doc="Name of the dark data product",
676  default="dark",
677  )
678 
679  # Camera-specific stray light removal.
680  doStrayLight = pexConfig.Field(
681  dtype=bool,
682  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
683  default=False,
684  )
685  strayLight = pexConfig.ConfigurableField(
686  target=StrayLightTask,
687  doc="y-band stray light correction"
688  )
689 
690  # Flat correction.
691  doFlat = pexConfig.Field(
692  dtype=bool,
693  doc="Apply flat field correction?",
694  default=True,
695  )
696  flatDataProductName = pexConfig.Field(
697  dtype=str,
698  doc="Name of the flat data product",
699  default="flat",
700  )
701  flatScalingType = pexConfig.ChoiceField(
702  dtype=str,
703  doc="The method for scaling the flat on the fly.",
704  default='USER',
705  allowed={
706  "USER": "Scale by flatUserScale",
707  "MEAN": "Scale by the inverse of the mean",
708  "MEDIAN": "Scale by the inverse of the median",
709  },
710  )
711  flatUserScale = pexConfig.Field(
712  dtype=float,
713  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
714  default=1.0,
715  )
716  doTweakFlat = pexConfig.Field(
717  dtype=bool,
718  doc="Tweak flats to match observed amplifier ratios?",
719  default=False
720  )
721 
722  # Amplifier normalization based on gains instead of using flats configuration.
723  doApplyGains = pexConfig.Field(
724  dtype=bool,
725  doc="Correct the amplifiers for their gains instead of applying flat correction",
726  default=False,
727  )
728  usePtcGains = pexConfig.Field(
729  dtype=bool,
730  doc="Use the gain values from the Photon Transfer Curve?",
731  default=False,
732  )
733  normalizeGains = pexConfig.Field(
734  dtype=bool,
735  doc="Normalize all the amplifiers in each CCD to have the same median value.",
736  default=False,
737  )
738 
739  # Fringe correction.
740  doFringe = pexConfig.Field(
741  dtype=bool,
742  doc="Apply fringe correction?",
743  default=True,
744  )
745  fringe = pexConfig.ConfigurableField(
746  target=FringeTask,
747  doc="Fringe subtraction task",
748  )
749  fringeAfterFlat = pexConfig.Field(
750  dtype=bool,
751  doc="Do fringe subtraction after flat-fielding?",
752  default=True,
753  )
754 
755  # Initial CCD-level background statistics options.
756  doMeasureBackground = pexConfig.Field(
757  dtype=bool,
758  doc="Measure the background level on the reduced image?",
759  default=False,
760  )
761 
762  # Camera-specific masking configuration.
763  doCameraSpecificMasking = pexConfig.Field(
764  dtype=bool,
765  doc="Mask camera-specific bad regions?",
766  default=False,
767  )
768  masking = pexConfig.ConfigurableField(
769  target=MaskingTask,
770  doc="Masking task."
771  )
772 
773  # Interpolation options.
774 
775  doInterpolate = pexConfig.Field(
776  dtype=bool,
777  doc="Interpolate masked pixels?",
778  default=True,
779  )
780  doSaturationInterpolation = pexConfig.Field(
781  dtype=bool,
782  doc="Perform interpolation over pixels masked as saturated?"
783  " NB: This is independent of doSaturation; if that is False this plane"
784  " will likely be blank, resulting in a no-op here.",
785  default=True,
786  )
787  doNanInterpolation = pexConfig.Field(
788  dtype=bool,
789  doc="Perform interpolation over pixels masked as NaN?"
790  " NB: This is independent of doNanMasking; if that is False this plane"
791  " will likely be blank, resulting in a no-op here.",
792  default=True,
793  )
794  doNanInterpAfterFlat = pexConfig.Field(
795  dtype=bool,
796  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
797  "also have to interpolate them before flat-fielding."),
798  default=False,
799  )
800  maskListToInterpolate = pexConfig.ListField(
801  dtype=str,
802  doc="List of mask planes that should be interpolated.",
803  default=['SAT', 'BAD'],
804  )
805  doSaveInterpPixels = pexConfig.Field(
806  dtype=bool,
807  doc="Save a copy of the pre-interpolated pixel values?",
808  default=False,
809  )
810 
811  # Default photometric calibration options.
812  fluxMag0T1 = pexConfig.DictField(
813  keytype=str,
814  itemtype=float,
815  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
816  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
817  ))
818  )
819  defaultFluxMag0T1 = pexConfig.Field(
820  dtype=float,
821  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
822  default=pow(10.0, 0.4*28.0)
823  )
824 
825  # Vignette correction configuration.
826  doVignette = pexConfig.Field(
827  dtype=bool,
828  doc="Apply vignetting parameters?",
829  default=False,
830  )
831  vignette = pexConfig.ConfigurableField(
832  target=VignetteTask,
833  doc="Vignetting task.",
834  )
835 
836  # Transmission curve configuration.
837  doAttachTransmissionCurve = pexConfig.Field(
838  dtype=bool,
839  default=False,
840  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
841  )
842  doUseOpticsTransmission = pexConfig.Field(
843  dtype=bool,
844  default=True,
845  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
846  )
847  doUseFilterTransmission = pexConfig.Field(
848  dtype=bool,
849  default=True,
850  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
851  )
852  doUseSensorTransmission = pexConfig.Field(
853  dtype=bool,
854  default=True,
855  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
856  )
857  doUseAtmosphereTransmission = pexConfig.Field(
858  dtype=bool,
859  default=True,
860  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
861  )
862 
863  # Illumination correction.
864  doIlluminationCorrection = pexConfig.Field(
865  dtype=bool,
866  default=False,
867  doc="Perform illumination correction?"
868  )
869  illuminationCorrectionDataProductName = pexConfig.Field(
870  dtype=str,
871  doc="Name of the illumination correction data product.",
872  default="illumcor",
873  )
874  illumScale = pexConfig.Field(
875  dtype=float,
876  doc="Scale factor for the illumination correction.",
877  default=1.0,
878  )
879  illumFilters = pexConfig.ListField(
880  dtype=str,
881  default=[],
882  doc="Only perform illumination correction for these filters."
883  )
884 
885  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
886  doWrite = pexConfig.Field(
887  dtype=bool,
888  doc="Persist postISRCCD?",
889  default=True,
890  )
891 
892  def validate(self):
893  super().validate()
894  if self.doFlatdoFlat and self.doApplyGainsdoApplyGains:
895  raise ValueError("You may not specify both doFlat and doApplyGains")
896  if self.doBiasBeforeOverscandoBiasBeforeOverscan and self.doTrimToMatchCalibdoTrimToMatchCalib:
897  raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
898  if self.doSaturationInterpolationdoSaturationInterpolation and self.saturatedMaskNamesaturatedMaskName not in self.maskListToInterpolatemaskListToInterpolate:
899  self.maskListToInterpolatemaskListToInterpolate.append(self.saturatedMaskNamesaturatedMaskName)
900  if not self.doSaturationInterpolationdoSaturationInterpolation and self.saturatedMaskNamesaturatedMaskName in self.maskListToInterpolatemaskListToInterpolate:
901  self.maskListToInterpolatemaskListToInterpolate.remove(self.saturatedMaskNamesaturatedMaskName)
902  if self.doNanInterpolationdoNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolatemaskListToInterpolate:
903  self.maskListToInterpolatemaskListToInterpolate.append("UNMASKEDNAN")
904 
905 
906 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
907  """Apply common instrument signature correction algorithms to a raw frame.
908 
909  The process for correcting imaging data is very similar from
910  camera to camera. This task provides a vanilla implementation of
911  doing these corrections, including the ability to turn certain
912  corrections off if they are not needed. The inputs to the primary
913  method, `run()`, are a raw exposure to be corrected and the
914  calibration data products. The raw input is a single chip sized
915  mosaic of all amps including overscans and other non-science
916  pixels. The method `runDataRef()` identifies and defines the
917  calibration data products, and is intended for use by a
918  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
919  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
920  subclassed for different camera, although the most camera specific
921  methods have been split into subtasks that can be redirected
922  appropriately.
923 
924  The __init__ method sets up the subtasks for ISR processing, using
925  the defaults from `lsst.ip.isr`.
926 
927  Parameters
928  ----------
929  args : `list`
930  Positional arguments passed to the Task constructor. None used at this time.
931  kwargs : `dict`, optional
932  Keyword arguments passed on to the Task constructor. None used at this time.
933  """
934  ConfigClass = IsrTaskConfig
935  _DefaultName = "isr"
936 
937  def __init__(self, **kwargs):
938  super().__init__(**kwargs)
939  self.makeSubtask("assembleCcd")
940  self.makeSubtask("crosstalk")
941  self.makeSubtask("strayLight")
942  self.makeSubtask("fringe")
943  self.makeSubtask("masking")
944  self.makeSubtask("overscan")
945  self.makeSubtask("vignette")
946 
947  def runQuantum(self, butlerQC, inputRefs, outputRefs):
948  inputs = butlerQC.get(inputRefs)
949 
950  try:
951  inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
952  except Exception as e:
953  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
954  (inputRefs, e))
955 
956  inputs['isGen3'] = True
957 
958  detector = inputs['ccdExposure'].getDetector()
959 
960  if self.config.doCrosstalk is True:
961  # Crosstalk sources need to be defined by the pipeline
962  # yaml if they exist.
963  if 'crosstalk' in inputs and inputs['crosstalk'] is not None:
964  if not isinstance(inputs['crosstalk'], CrosstalkCalib):
965  inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk'])
966  else:
967  coeffVector = (self.config.crosstalk.crosstalkValues
968  if self.config.crosstalk.useConfigCoefficients else None)
969  crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
970  inputs['crosstalk'] = crosstalkCalib
971  if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0:
972  if 'crosstalkSources' not in inputs:
973  self.log.warn("No crosstalkSources found for chip with interChip terms!")
974 
975  if self.doLinearizedoLinearize(detector) is True:
976  if 'linearizer' in inputs:
977  if isinstance(inputs['linearizer'], dict):
978  linearizer = linearize.Linearizer(detector=detector, log=self.log)
979  linearizer.fromYaml(inputs['linearizer'])
980  self.log.warn("Dictionary linearizers will be deprecated in DM-28741.")
981  elif isinstance(inputs['linearizer'], numpy.ndarray):
982  linearizer = linearize.Linearizer(table=inputs.get('linearizer', None),
983  detector=detector,
984  log=self.log)
985  self.log.warn("Bare lookup table linearizers will be deprecated in DM-28741.")
986  else:
987  linearizer = inputs['linearizer']
988  linearizer.log = self.log
989  inputs['linearizer'] = linearizer
990  else:
991  inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log)
992  self.log.warn("Constructing linearizer from cameraGeom information.")
993 
994  if self.config.doDefect is True:
995  if "defects" in inputs and inputs['defects'] is not None:
996  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
997  # masking expects a list of defects defined by their bounding box
998  if not isinstance(inputs["defects"], Defects):
999  inputs["defects"] = Defects.fromTable(inputs["defects"])
1000 
1001  # Load the correct style of brighter-fatter kernel, and repack
1002  # the information as a numpy array.
1003  if self.config.doBrighterFatter:
1004  brighterFatterKernel = inputs.pop('newBFKernel', None)
1005  if brighterFatterKernel is None:
1006  brighterFatterKernel = inputs.get('bfKernel', None)
1007 
1008  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1009  # This is a ISR calib kernel
1010  detName = detector.getName()
1011  level = brighterFatterKernel.level
1012 
1013  # This is expected to be a dictionary of amp-wise gains.
1014  inputs['bfGains'] = brighterFatterKernel.gain
1015  if self.config.brighterFatterLevel == 'DETECTOR':
1016  if level == 'DETECTOR':
1017  if detName in brighterFatterKernel.detKernels:
1018  inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1019  else:
1020  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1021  elif level == 'AMP':
1022  self.log.warn("Making DETECTOR level kernel from AMP based brighter fatter kernels.")
1023  brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1024  inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1025  elif self.config.brighterFatterLevel == 'AMP':
1026  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1027 
1028  if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
1029  expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
1030  inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
1031  expId=expId,
1032  assembler=self.assembleCcd
1033  if self.config.doAssembleIsrExposures else None)
1034  else:
1035  inputs['fringes'] = pipeBase.Struct(fringes=None)
1036 
1037  if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
1038  if 'strayLightData' not in inputs:
1039  inputs['strayLightData'] = None
1040 
1041  outputs = self.runrun(**inputs)
1042  butlerQC.put(outputs, outputRefs)
1043 
1044  def readIsrData(self, dataRef, rawExposure):
1045  """Retrieve necessary frames for instrument signature removal.
1046 
1047  Pre-fetching all required ISR data products limits the IO
1048  required by the ISR. Any conflict between the calibration data
1049  available and that needed for ISR is also detected prior to
1050  doing processing, allowing it to fail quickly.
1051 
1052  Parameters
1053  ----------
1054  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1055  Butler reference of the detector data to be processed
1056  rawExposure : `afw.image.Exposure`
1057  The raw exposure that will later be corrected with the
1058  retrieved calibration data; should not be modified in this
1059  method.
1060 
1061  Returns
1062  -------
1063  result : `lsst.pipe.base.Struct`
1064  Result struct with components (which may be `None`):
1065  - ``bias``: bias calibration frame (`afw.image.Exposure`)
1066  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
1067  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1068  - ``dark``: dark calibration frame (`afw.image.Exposure`)
1069  - ``flat``: flat calibration frame (`afw.image.Exposure`)
1070  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1071  - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1072  - ``fringes``: `lsst.pipe.base.Struct` with components:
1073  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1074  - ``seed``: random seed derived from the ccdExposureId for random
1075  number generator (`uint32`).
1076  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1077  A ``TransmissionCurve`` that represents the throughput of the optics,
1078  to be evaluated in focal-plane coordinates.
1079  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1080  A ``TransmissionCurve`` that represents the throughput of the filter
1081  itself, to be evaluated in focal-plane coordinates.
1082  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1083  A ``TransmissionCurve`` that represents the throughput of the sensor
1084  itself, to be evaluated in post-assembly trimmed detector coordinates.
1085  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1086  A ``TransmissionCurve`` that represents the throughput of the
1087  atmosphere, assumed to be spatially constant.
1088  - ``strayLightData`` : `object`
1089  An opaque object containing calibration information for
1090  stray-light correction. If `None`, no correction will be
1091  performed.
1092  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1093 
1094  Raises
1095  ------
1096  NotImplementedError :
1097  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1098  """
1099  try:
1100  dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1101  dateObs = dateObs.toPython().isoformat()
1102  except RuntimeError:
1103  self.log.warn("Unable to identify dateObs for rawExposure.")
1104  dateObs = None
1105 
1106  ccd = rawExposure.getDetector()
1107  filterLabel = rawExposure.getFilterLabel()
1108  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1109  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
1110  biasExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.biasDataProductName)
1111  if self.config.doBias else None)
1112  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
1113  linearizer = (dataRef.get("linearizer", immediate=True)
1114  if self.doLinearizedoLinearize(ccd) else None)
1115  if linearizer is not None and not isinstance(linearizer, numpy.ndarray):
1116  linearizer.log = self.log
1117  if isinstance(linearizer, numpy.ndarray):
1118  linearizer = linearize.Linearizer(table=linearizer, detector=ccd)
1119 
1120  crosstalkCalib = None
1121  if self.config.doCrosstalk:
1122  try:
1123  crosstalkCalib = dataRef.get("crosstalk", immediate=True)
1124  except NoResults:
1125  coeffVector = (self.config.crosstalk.crosstalkValues
1126  if self.config.crosstalk.useConfigCoefficients else None)
1127  crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1128  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1129  if self.config.doCrosstalk else None)
1130 
1131  darkExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.darkDataProductName)
1132  if self.config.doDark else None)
1133  flatExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.flatDataProductName,
1134  dateObs=dateObs)
1135  if self.config.doFlat else None)
1136 
1137  brighterFatterKernel = None
1138  brighterFatterGains = None
1139  if self.config.doBrighterFatter is True:
1140  try:
1141  # Use the new-style cp_pipe version of the kernel if it exists
1142  # If using a new-style kernel, always use the self-consistent
1143  # gains, i.e. the ones inside the kernel object itself
1144  brighterFatterKernel = dataRef.get("brighterFatterKernel")
1145  brighterFatterGains = brighterFatterKernel.gain
1146  self.log.info("New style brighter-fatter kernel (brighterFatterKernel) loaded")
1147  except NoResults:
1148  try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
1149  brighterFatterKernel = dataRef.get("bfKernel")
1150  self.log.info("Old style brighter-fatter kernel (np.array) loaded")
1151  except NoResults:
1152  brighterFatterKernel = None
1153  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1154  # If the kernel is not an ndarray, it's the cp_pipe version
1155  # so extract the kernel for this detector, or raise an error
1156  if self.config.brighterFatterLevel == 'DETECTOR':
1157  if brighterFatterKernel.detectorKernel:
1158  brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
1159  else:
1160  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1161  else:
1162  # TODO DM-15631 for implementing this
1163  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1164 
1165  defectList = (dataRef.get("defects")
1166  if self.config.doDefect else None)
1167  expId = rawExposure.getInfo().getVisitInfo().getExposureId()
1168  fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1169  if self.config.doAssembleIsrExposures else None)
1170  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1171  else pipeBase.Struct(fringes=None))
1172 
1173  if self.config.doAttachTransmissionCurve:
1174  opticsTransmission = (dataRef.get("transmission_optics")
1175  if self.config.doUseOpticsTransmission else None)
1176  filterTransmission = (dataRef.get("transmission_filter")
1177  if self.config.doUseFilterTransmission else None)
1178  sensorTransmission = (dataRef.get("transmission_sensor")
1179  if self.config.doUseSensorTransmission else None)
1180  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1181  if self.config.doUseAtmosphereTransmission else None)
1182  else:
1183  opticsTransmission = None
1184  filterTransmission = None
1185  sensorTransmission = None
1186  atmosphereTransmission = None
1187 
1188  if self.config.doStrayLight:
1189  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1190  else:
1191  strayLightData = None
1192 
1193  illumMaskedImage = (self.getIsrExposuregetIsrExposure(dataRef,
1194  self.config.illuminationCorrectionDataProductName).getMaskedImage()
1195  if (self.config.doIlluminationCorrection
1196  and physicalFilter in self.config.illumFilters)
1197  else None)
1198 
1199  # Struct should include only kwargs to run()
1200  return pipeBase.Struct(bias=biasExposure,
1201  linearizer=linearizer,
1202  crosstalk=crosstalkCalib,
1203  crosstalkSources=crosstalkSources,
1204  dark=darkExposure,
1205  flat=flatExposure,
1206  bfKernel=brighterFatterKernel,
1207  bfGains=brighterFatterGains,
1208  defects=defectList,
1209  fringes=fringeStruct,
1210  opticsTransmission=opticsTransmission,
1211  filterTransmission=filterTransmission,
1212  sensorTransmission=sensorTransmission,
1213  atmosphereTransmission=atmosphereTransmission,
1214  strayLightData=strayLightData,
1215  illumMaskedImage=illumMaskedImage
1216  )
1217 
1218  @pipeBase.timeMethod
1219  def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1220  crosstalk=None, crosstalkSources=None,
1221  dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1222  fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1223  sensorTransmission=None, atmosphereTransmission=None,
1224  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1225  isGen3=False,
1226  ):
1227  """Perform instrument signature removal on an exposure.
1228 
1229  Steps included in the ISR processing, in order performed, are:
1230  - saturation and suspect pixel masking
1231  - overscan subtraction
1232  - CCD assembly of individual amplifiers
1233  - bias subtraction
1234  - variance image construction
1235  - linearization of non-linear response
1236  - crosstalk masking
1237  - brighter-fatter correction
1238  - dark subtraction
1239  - fringe correction
1240  - stray light subtraction
1241  - flat correction
1242  - masking of known defects and camera specific features
1243  - vignette calculation
1244  - appending transmission curve and distortion model
1245 
1246  Parameters
1247  ----------
1248  ccdExposure : `lsst.afw.image.Exposure`
1249  The raw exposure that is to be run through ISR. The
1250  exposure is modified by this method.
1251  camera : `lsst.afw.cameraGeom.Camera`, optional
1252  The camera geometry for this exposure. Required if ``isGen3`` is
1253  `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1254  ``flat`` does not have an associated detector.
1255  bias : `lsst.afw.image.Exposure`, optional
1256  Bias calibration frame.
1257  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1258  Functor for linearization.
1259  crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1260  Calibration for crosstalk.
1261  crosstalkSources : `list`, optional
1262  List of possible crosstalk sources.
1263  dark : `lsst.afw.image.Exposure`, optional
1264  Dark calibration frame.
1265  flat : `lsst.afw.image.Exposure`, optional
1266  Flat calibration frame.
1267  ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
1268  Photon transfer curve dataset, with, e.g., gains
1269  and read noise.
1270  bfKernel : `numpy.ndarray`, optional
1271  Brighter-fatter kernel.
1272  bfGains : `dict` of `float`, optional
1273  Gains used to override the detector's nominal gains for the
1274  brighter-fatter correction. A dict keyed by amplifier name for
1275  the detector in question.
1276  defects : `lsst.ip.isr.Defects`, optional
1277  List of defects.
1278  fringes : `lsst.pipe.base.Struct`, optional
1279  Struct containing the fringe correction data, with
1280  elements:
1281  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1282  - ``seed``: random seed derived from the ccdExposureId for random
1283  number generator (`uint32`)
1284  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1285  A ``TransmissionCurve`` that represents the throughput of the optics,
1286  to be evaluated in focal-plane coordinates.
1287  filterTransmission : `lsst.afw.image.TransmissionCurve`
1288  A ``TransmissionCurve`` that represents the throughput of the filter
1289  itself, to be evaluated in focal-plane coordinates.
1290  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1291  A ``TransmissionCurve`` that represents the throughput of the sensor
1292  itself, to be evaluated in post-assembly trimmed detector coordinates.
1293  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1294  A ``TransmissionCurve`` that represents the throughput of the
1295  atmosphere, assumed to be spatially constant.
1296  detectorNum : `int`, optional
1297  The integer number for the detector to process.
1298  isGen3 : bool, optional
1299  Flag this call to run() as using the Gen3 butler environment.
1300  strayLightData : `object`, optional
1301  Opaque object containing calibration information for stray-light
1302  correction. If `None`, no correction will be performed.
1303  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1304  Illumination correction image.
1305 
1306  Returns
1307  -------
1308  result : `lsst.pipe.base.Struct`
1309  Result struct with component:
1310  - ``exposure`` : `afw.image.Exposure`
1311  The fully ISR corrected exposure.
1312  - ``outputExposure`` : `afw.image.Exposure`
1313  An alias for `exposure`
1314  - ``ossThumb`` : `numpy.ndarray`
1315  Thumbnail image of the exposure after overscan subtraction.
1316  - ``flattenedThumb`` : `numpy.ndarray`
1317  Thumbnail image of the exposure after flat-field correction.
1318 
1319  Raises
1320  ------
1321  RuntimeError
1322  Raised if a configuration option is set to True, but the
1323  required calibration data has not been specified.
1324 
1325  Notes
1326  -----
1327  The current processed exposure can be viewed by setting the
1328  appropriate lsstDebug entries in the `debug.display`
1329  dictionary. The names of these entries correspond to some of
1330  the IsrTaskConfig Boolean options, with the value denoting the
1331  frame to use. The exposure is shown inside the matching
1332  option check and after the processing of that step has
1333  finished. The steps with debug points are:
1334 
1335  doAssembleCcd
1336  doBias
1337  doCrosstalk
1338  doBrighterFatter
1339  doDark
1340  doFringe
1341  doStrayLight
1342  doFlat
1343 
1344  In addition, setting the "postISRCCD" entry displays the
1345  exposure after all ISR processing has finished.
1346 
1347  """
1348 
1349  if isGen3 is True:
1350  # Gen3 currently cannot automatically do configuration overrides.
1351  # DM-15257 looks to discuss this issue.
1352  # Configure input exposures;
1353  if detectorNum is None:
1354  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1355 
1356  ccdExposure = self.ensureExposureensureExposure(ccdExposure, camera, detectorNum)
1357  bias = self.ensureExposureensureExposure(bias, camera, detectorNum)
1358  dark = self.ensureExposureensureExposure(dark, camera, detectorNum)
1359  flat = self.ensureExposureensureExposure(flat, camera, detectorNum)
1360  else:
1361  if isinstance(ccdExposure, ButlerDataRef):
1362  return self.runDataRefrunDataRef(ccdExposure)
1363 
1364  ccd = ccdExposure.getDetector()
1365  filterLabel = ccdExposure.getFilterLabel()
1366  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1367 
1368  if not ccd:
1369  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1370  ccd = [FakeAmp(ccdExposure, self.config)]
1371 
1372  # Validate Input
1373  if self.config.doBias and bias is None:
1374  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1375  if self.doLinearizedoLinearize(ccd) and linearizer is None:
1376  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1377  if self.config.doBrighterFatter and bfKernel is None:
1378  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1379  if self.config.doDark and dark is None:
1380  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1381  if self.config.doFlat and flat is None:
1382  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1383  if self.config.doDefect and defects is None:
1384  raise RuntimeError("Must supply defects if config.doDefect=True.")
1385  if (self.config.doFringe and physicalFilter in self.fringe.config.filters
1386  and fringes.fringes is None):
1387  # The `fringes` object needs to be a pipeBase.Struct, as
1388  # we use it as a `dict` for the parameters of
1389  # `FringeTask.run()`. The `fringes.fringes` `list` may
1390  # not be `None` if `doFringe=True`. Otherwise, raise.
1391  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1392  if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters
1393  and illumMaskedImage is None):
1394  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1395 
1396  # Begin ISR processing.
1397  if self.config.doConvertIntToFloat:
1398  self.log.info("Converting exposure to floating point values.")
1399  ccdExposure = self.convertIntToFloatconvertIntToFloat(ccdExposure)
1400 
1401  if self.config.doBias and self.config.doBiasBeforeOverscan:
1402  self.log.info("Applying bias correction.")
1403  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1404  trimToFit=self.config.doTrimToMatchCalib)
1405  self.debugViewdebugView(ccdExposure, "doBias")
1406 
1407  # Amplifier level processing.
1408  overscans = []
1409  for amp in ccd:
1410  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1411  if ccdExposure.getBBox().contains(amp.getBBox()):
1412  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1413  badAmp = self.maskAmplifiermaskAmplifier(ccdExposure, amp, defects)
1414 
1415  if self.config.doOverscan and not badAmp:
1416  # Overscan correction on amp-by-amp basis.
1417  overscanResults = self.overscanCorrectionoverscanCorrection(ccdExposure, amp)
1418  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1419  if overscanResults is not None and \
1420  self.config.qa is not None and self.config.qa.saveStats is True:
1421  if isinstance(overscanResults.overscanFit, float):
1422  qaMedian = overscanResults.overscanFit
1423  qaStdev = float("NaN")
1424  else:
1425  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1426  afwMath.MEDIAN | afwMath.STDEVCLIP)
1427  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1428  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1429 
1430  self.metadata.set(f"FIT MEDIAN {amp.getName()}", qaMedian)
1431  self.metadata.set(f"FIT STDEV {amp.getName()}", qaStdev)
1432  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1433  amp.getName(), qaMedian, qaStdev)
1434 
1435  # Residuals after overscan correction
1436  qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1437  afwMath.MEDIAN | afwMath.STDEVCLIP)
1438  qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1439  qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1440 
1441  self.metadata.set(f"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter)
1442  self.metadata.set(f"RESIDUAL STDEV {amp.getName()}", qaStdevAfter)
1443  self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f",
1444  amp.getName(), qaMedianAfter, qaStdevAfter)
1445 
1446  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1447  else:
1448  if badAmp:
1449  self.log.warn("Amplifier %s is bad.", amp.getName())
1450  overscanResults = None
1451 
1452  overscans.append(overscanResults if overscanResults is not None else None)
1453  else:
1454  self.log.info("Skipped OSCAN for %s.", amp.getName())
1455 
1456  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1457  self.log.info("Applying crosstalk correction.")
1458  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1459  crosstalkSources=crosstalkSources, camera=camera)
1460  self.debugViewdebugView(ccdExposure, "doCrosstalk")
1461 
1462  if self.config.doAssembleCcd:
1463  self.log.info("Assembling CCD from amplifiers.")
1464  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1465 
1466  if self.config.expectWcs and not ccdExposure.getWcs():
1467  self.log.warn("No WCS found in input exposure.")
1468  self.debugViewdebugView(ccdExposure, "doAssembleCcd")
1469 
1470  ossThumb = None
1471  if self.config.qa.doThumbnailOss:
1472  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1473 
1474  if self.config.doBias and not self.config.doBiasBeforeOverscan:
1475  self.log.info("Applying bias correction.")
1476  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1477  trimToFit=self.config.doTrimToMatchCalib)
1478  self.debugViewdebugView(ccdExposure, "doBias")
1479 
1480  if self.config.doVariance:
1481  for amp, overscanResults in zip(ccd, overscans):
1482  if ccdExposure.getBBox().contains(amp.getBBox()):
1483  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1484  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1485  if overscanResults is not None:
1486  self.updateVarianceupdateVariance(ampExposure, amp,
1487  overscanImage=overscanResults.overscanImage,
1488  ptcDataset=ptc)
1489  else:
1490  self.updateVarianceupdateVariance(ampExposure, amp,
1491  overscanImage=None,
1492  ptcDataset=ptc)
1493  if self.config.qa is not None and self.config.qa.saveStats is True:
1494  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1495  afwMath.MEDIAN | afwMath.STDEVCLIP)
1496  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1497  qaStats.getValue(afwMath.MEDIAN))
1498  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1499  qaStats.getValue(afwMath.STDEVCLIP))
1500  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1501  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1502  qaStats.getValue(afwMath.STDEVCLIP))
1503 
1504  if self.doLinearizedoLinearize(ccd):
1505  self.log.info("Applying linearizer.")
1506  linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1507  detector=ccd, log=self.log)
1508 
1509  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1510  self.log.info("Applying crosstalk correction.")
1511  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1512  crosstalkSources=crosstalkSources, isTrimmed=True)
1513  self.debugViewdebugView(ccdExposure, "doCrosstalk")
1514 
1515  # Masking block. Optionally mask known defects, NAN/inf pixels, widen trails, and do
1516  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1517  if self.config.doDefect:
1518  self.log.info("Masking defects.")
1519  self.maskDefectmaskDefect(ccdExposure, defects)
1520 
1521  if self.config.numEdgeSuspect > 0:
1522  self.log.info("Masking edges as SUSPECT.")
1523  self.maskEdgesmaskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1524  maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
1525 
1526  if self.config.doNanMasking:
1527  self.log.info("Masking non-finite (NAN, inf) value pixels.")
1528  self.maskNanmaskNan(ccdExposure)
1529 
1530  if self.config.doWidenSaturationTrails:
1531  self.log.info("Widening saturation trails.")
1532  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1533 
1534  if self.config.doCameraSpecificMasking:
1535  self.log.info("Masking regions for camera specific reasons.")
1536  self.masking.run(ccdExposure)
1537 
1538  if self.config.doBrighterFatter:
1539  # We need to apply flats and darks before we can interpolate, and we
1540  # need to interpolate before we do B-F, but we do B-F without the
1541  # flats and darks applied so we can work in units of electrons or holes.
1542  # This context manager applies and then removes the darks and flats.
1543  #
1544  # We also do not want to interpolate values here, so operate on temporary
1545  # images so we can apply only the BF-correction and roll back the
1546  # interpolation.
1547  interpExp = ccdExposure.clone()
1548  with self.flatContextflatContext(interpExp, flat, dark):
1549  isrFunctions.interpolateFromMask(
1550  maskedImage=interpExp.getMaskedImage(),
1551  fwhm=self.config.fwhm,
1552  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1553  maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1554  )
1555  bfExp = interpExp.clone()
1556 
1557  self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.",
1558  type(bfKernel), type(bfGains))
1559  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1560  self.config.brighterFatterMaxIter,
1561  self.config.brighterFatterThreshold,
1562  self.config.brighterFatterApplyGain,
1563  bfGains)
1564  if bfResults[1] == self.config.brighterFatterMaxIter:
1565  self.log.warn("Brighter-fatter correction did not converge, final difference %f.",
1566  bfResults[0])
1567  else:
1568  self.log.info("Finished brighter-fatter correction in %d iterations.",
1569  bfResults[1])
1570  image = ccdExposure.getMaskedImage().getImage()
1571  bfCorr = bfExp.getMaskedImage().getImage()
1572  bfCorr -= interpExp.getMaskedImage().getImage()
1573  image += bfCorr
1574 
1575  # Applying the brighter-fatter correction applies a
1576  # convolution to the science image. At the edges this
1577  # convolution may not have sufficient valid pixels to
1578  # produce a valid correction. Mark pixels within the size
1579  # of the brighter-fatter kernel as EDGE to warn of this
1580  # fact.
1581  self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1582  self.maskEdgesmaskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1583  maskPlane="EDGE")
1584 
1585  if self.config.brighterFatterMaskGrowSize > 0:
1586  self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1587  for maskPlane in self.config.brighterFatterMaskListToInterpolate:
1588  isrFunctions.growMasks(ccdExposure.getMask(),
1589  radius=self.config.brighterFatterMaskGrowSize,
1590  maskNameList=maskPlane,
1591  maskValue=maskPlane)
1592 
1593  self.debugViewdebugView(ccdExposure, "doBrighterFatter")
1594 
1595  if self.config.doDark:
1596  self.log.info("Applying dark correction.")
1597  self.darkCorrectiondarkCorrection(ccdExposure, dark)
1598  self.debugViewdebugView(ccdExposure, "doDark")
1599 
1600  if self.config.doFringe and not self.config.fringeAfterFlat:
1601  self.log.info("Applying fringe correction before flat.")
1602  self.fringe.run(ccdExposure, **fringes.getDict())
1603  self.debugViewdebugView(ccdExposure, "doFringe")
1604 
1605  if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1606  self.log.info("Checking strayLight correction.")
1607  self.strayLight.run(ccdExposure, strayLightData)
1608  self.debugViewdebugView(ccdExposure, "doStrayLight")
1609 
1610  if self.config.doFlat:
1611  self.log.info("Applying flat correction.")
1612  self.flatCorrectionflatCorrection(ccdExposure, flat)
1613  self.debugViewdebugView(ccdExposure, "doFlat")
1614 
1615  if self.config.doApplyGains:
1616  self.log.info("Applying gain correction instead of flat.")
1617  if self.config.usePtcGains:
1618  self.log.info("Using gains from the Photon Transfer Curve.")
1619  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1620  ptcGains=ptc.gain)
1621  else:
1622  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1623 
1624  if self.config.doFringe and self.config.fringeAfterFlat:
1625  self.log.info("Applying fringe correction after flat.")
1626  self.fringe.run(ccdExposure, **fringes.getDict())
1627 
1628  if self.config.doVignette:
1629  self.log.info("Constructing Vignette polygon.")
1630  self.vignettePolygonvignettePolygon = self.vignette.run(ccdExposure)
1631 
1632  if self.config.vignette.doWriteVignettePolygon:
1633  self.setValidPolygonIntersectsetValidPolygonIntersect(ccdExposure, self.vignettePolygonvignettePolygon)
1634 
1635  if self.config.doAttachTransmissionCurve:
1636  self.log.info("Adding transmission curves.")
1637  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1638  filterTransmission=filterTransmission,
1639  sensorTransmission=sensorTransmission,
1640  atmosphereTransmission=atmosphereTransmission)
1641 
1642  flattenedThumb = None
1643  if self.config.qa.doThumbnailFlattened:
1644  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1645 
1646  if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters:
1647  self.log.info("Performing illumination correction.")
1648  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1649  illumMaskedImage, illumScale=self.config.illumScale,
1650  trimToFit=self.config.doTrimToMatchCalib)
1651 
1652  preInterpExp = None
1653  if self.config.doSaveInterpPixels:
1654  preInterpExp = ccdExposure.clone()
1655 
1656  # Reset and interpolate bad pixels.
1657  #
1658  # Large contiguous bad regions (which should have the BAD mask
1659  # bit set) should have their values set to the image median.
1660  # This group should include defects and bad amplifiers. As the
1661  # area covered by these defects are large, there's little
1662  # reason to expect that interpolation would provide a more
1663  # useful value.
1664  #
1665  # Smaller defects can be safely interpolated after the larger
1666  # regions have had their pixel values reset. This ensures
1667  # that the remaining defects adjacent to bad amplifiers (as an
1668  # example) do not attempt to interpolate extreme values.
1669  if self.config.doSetBadRegions:
1670  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1671  if badPixelCount > 0:
1672  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1673 
1674  if self.config.doInterpolate:
1675  self.log.info("Interpolating masked pixels.")
1676  isrFunctions.interpolateFromMask(
1677  maskedImage=ccdExposure.getMaskedImage(),
1678  fwhm=self.config.fwhm,
1679  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1680  maskNameList=list(self.config.maskListToInterpolate)
1681  )
1682 
1683  self.roughZeroPointroughZeroPoint(ccdExposure)
1684 
1685  if self.config.doMeasureBackground:
1686  self.log.info("Measuring background level.")
1687  self.measureBackgroundmeasureBackground(ccdExposure, self.config.qa)
1688 
1689  if self.config.qa is not None and self.config.qa.saveStats is True:
1690  for amp in ccd:
1691  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1692  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1693  afwMath.MEDIAN | afwMath.STDEVCLIP)
1694  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1695  qaStats.getValue(afwMath.MEDIAN))
1696  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1697  qaStats.getValue(afwMath.STDEVCLIP))
1698  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1699  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1700  qaStats.getValue(afwMath.STDEVCLIP))
1701 
1702  self.debugViewdebugView(ccdExposure, "postISRCCD")
1703 
1704  return pipeBase.Struct(
1705  exposure=ccdExposure,
1706  ossThumb=ossThumb,
1707  flattenedThumb=flattenedThumb,
1708 
1709  preInterpolatedExposure=preInterpExp,
1710  outputExposure=ccdExposure,
1711  outputOssThumbnail=ossThumb,
1712  outputFlattenedThumbnail=flattenedThumb,
1713  )
1714 
1715  @pipeBase.timeMethod
1716  def runDataRef(self, sensorRef):
1717  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1718 
1719  This method contains the `CmdLineTask` interface to the ISR
1720  processing. All IO is handled here, freeing the `run()` method
1721  to manage only pixel-level calculations. The steps performed
1722  are:
1723  - Read in necessary detrending/isr/calibration data.
1724  - Process raw exposure in `run()`.
1725  - Persist the ISR-corrected exposure as "postISRCCD" if
1726  config.doWrite=True.
1727 
1728  Parameters
1729  ----------
1730  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1731  DataRef of the detector data to be processed
1732 
1733  Returns
1734  -------
1735  result : `lsst.pipe.base.Struct`
1736  Result struct with component:
1737  - ``exposure`` : `afw.image.Exposure`
1738  The fully ISR corrected exposure.
1739 
1740  Raises
1741  ------
1742  RuntimeError
1743  Raised if a configuration option is set to True, but the
1744  required calibration data does not exist.
1745 
1746  """
1747  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1748 
1749  ccdExposure = sensorRef.get(self.config.datasetType)
1750 
1751  camera = sensorRef.get("camera")
1752  isrData = self.readIsrDatareadIsrData(sensorRef, ccdExposure)
1753 
1754  result = self.runrun(ccdExposure, camera=camera, **isrData.getDict())
1755 
1756  if self.config.doWrite:
1757  sensorRef.put(result.exposure, "postISRCCD")
1758  if result.preInterpolatedExposure is not None:
1759  sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1760  if result.ossThumb is not None:
1761  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1762  if result.flattenedThumb is not None:
1763  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1764 
1765  return result
1766 
1767  def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1768  """Retrieve a calibration dataset for removing instrument signature.
1769 
1770  Parameters
1771  ----------
1772 
1773  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1774  DataRef of the detector data to find calibration datasets
1775  for.
1776  datasetType : `str`
1777  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1778  dateObs : `str`, optional
1779  Date of the observation. Used to correct butler failures
1780  when using fallback filters.
1781  immediate : `Bool`
1782  If True, disable butler proxies to enable error handling
1783  within this routine.
1784 
1785  Returns
1786  -------
1787  exposure : `lsst.afw.image.Exposure`
1788  Requested calibration frame.
1789 
1790  Raises
1791  ------
1792  RuntimeError
1793  Raised if no matching calibration frame can be found.
1794  """
1795  try:
1796  exp = dataRef.get(datasetType, immediate=immediate)
1797  except Exception as exc1:
1798  if not self.config.fallbackFilterName:
1799  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1800  try:
1801  if self.config.useFallbackDate and dateObs:
1802  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1803  dateObs=dateObs, immediate=immediate)
1804  else:
1805  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1806  except Exception as exc2:
1807  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1808  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1809  self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1810 
1811  if self.config.doAssembleIsrExposures:
1812  exp = self.assembleCcd.assembleCcd(exp)
1813  return exp
1814 
1815  def ensureExposure(self, inputExp, camera, detectorNum):
1816  """Ensure that the data returned by Butler is a fully constructed exposure.
1817 
1818  ISR requires exposure-level image data for historical reasons, so if we did
1819  not recieve that from Butler, construct it from what we have, modifying the
1820  input in place.
1821 
1822  Parameters
1823  ----------
1824  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1825  `lsst.afw.image.ImageF`
1826  The input data structure obtained from Butler.
1827  camera : `lsst.afw.cameraGeom.camera`
1828  The camera associated with the image. Used to find the appropriate
1829  detector.
1830  detectorNum : `int`
1831  The detector this exposure should match.
1832 
1833  Returns
1834  -------
1835  inputExp : `lsst.afw.image.Exposure`
1836  The re-constructed exposure, with appropriate detector parameters.
1837 
1838  Raises
1839  ------
1840  TypeError
1841  Raised if the input data cannot be used to construct an exposure.
1842  """
1843  if isinstance(inputExp, afwImage.DecoratedImageU):
1844  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1845  elif isinstance(inputExp, afwImage.ImageF):
1846  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1847  elif isinstance(inputExp, afwImage.MaskedImageF):
1848  inputExp = afwImage.makeExposure(inputExp)
1849  elif isinstance(inputExp, afwImage.Exposure):
1850  pass
1851  elif inputExp is None:
1852  # Assume this will be caught by the setup if it is a problem.
1853  return inputExp
1854  else:
1855  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1856  (type(inputExp), ))
1857 
1858  if inputExp.getDetector() is None:
1859  inputExp.setDetector(camera[detectorNum])
1860 
1861  return inputExp
1862 
1863  def convertIntToFloat(self, exposure):
1864  """Convert exposure image from uint16 to float.
1865 
1866  If the exposure does not need to be converted, the input is
1867  immediately returned. For exposures that are converted to use
1868  floating point pixels, the variance is set to unity and the
1869  mask to zero.
1870 
1871  Parameters
1872  ----------
1873  exposure : `lsst.afw.image.Exposure`
1874  The raw exposure to be converted.
1875 
1876  Returns
1877  -------
1878  newexposure : `lsst.afw.image.Exposure`
1879  The input ``exposure``, converted to floating point pixels.
1880 
1881  Raises
1882  ------
1883  RuntimeError
1884  Raised if the exposure type cannot be converted to float.
1885 
1886  """
1887  if isinstance(exposure, afwImage.ExposureF):
1888  # Nothing to be done
1889  self.log.debug("Exposure already of type float.")
1890  return exposure
1891  if not hasattr(exposure, "convertF"):
1892  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1893 
1894  newexposure = exposure.convertF()
1895  newexposure.variance[:] = 1
1896  newexposure.mask[:] = 0x0
1897 
1898  return newexposure
1899 
1900  def maskAmplifier(self, ccdExposure, amp, defects):
1901  """Identify bad amplifiers, saturated and suspect pixels.
1902 
1903  Parameters
1904  ----------
1905  ccdExposure : `lsst.afw.image.Exposure`
1906  Input exposure to be masked.
1907  amp : `lsst.afw.table.AmpInfoCatalog`
1908  Catalog of parameters defining the amplifier on this
1909  exposure to mask.
1910  defects : `lsst.ip.isr.Defects`
1911  List of defects. Used to determine if the entire
1912  amplifier is bad.
1913 
1914  Returns
1915  -------
1916  badAmp : `Bool`
1917  If this is true, the entire amplifier area is covered by
1918  defects and unusable.
1919 
1920  """
1921  maskedImage = ccdExposure.getMaskedImage()
1922 
1923  badAmp = False
1924 
1925  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1926  # comparison with current defects definition.
1927  if defects is not None:
1928  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1929 
1930  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1931  # association with pixels in current ccdExposure).
1932  if badAmp:
1933  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1934  afwImage.PARENT)
1935  maskView = dataView.getMask()
1936  maskView |= maskView.getPlaneBitMask("BAD")
1937  del maskView
1938  return badAmp
1939 
1940  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1941  # Saturation and suspect pixels can be masked now, though.
1942  limits = dict()
1943  if self.config.doSaturation and not badAmp:
1944  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1945  if self.config.doSuspect and not badAmp:
1946  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1947  if math.isfinite(self.config.saturation):
1948  limits.update({self.config.saturatedMaskName: self.config.saturation})
1949 
1950  for maskName, maskThreshold in limits.items():
1951  if not math.isnan(maskThreshold):
1952  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1953  isrFunctions.makeThresholdMask(
1954  maskedImage=dataView,
1955  threshold=maskThreshold,
1956  growFootprints=0,
1957  maskName=maskName
1958  )
1959 
1960  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1961  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1962  afwImage.PARENT)
1963  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1964  self.config.suspectMaskName])
1965  if numpy.all(maskView.getArray() & maskVal > 0):
1966  badAmp = True
1967  maskView |= maskView.getPlaneBitMask("BAD")
1968 
1969  return badAmp
1970 
1971  def overscanCorrection(self, ccdExposure, amp):
1972  """Apply overscan correction in place.
1973 
1974  This method does initial pixel rejection of the overscan
1975  region. The overscan can also be optionally segmented to
1976  allow for discontinuous overscan responses to be fit
1977  separately. The actual overscan subtraction is performed by
1978  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1979  which is called here after the amplifier is preprocessed.
1980 
1981  Parameters
1982  ----------
1983  ccdExposure : `lsst.afw.image.Exposure`
1984  Exposure to have overscan correction performed.
1985  amp : `lsst.afw.cameraGeom.Amplifer`
1986  The amplifier to consider while correcting the overscan.
1987 
1988  Returns
1989  -------
1990  overscanResults : `lsst.pipe.base.Struct`
1991  Result struct with components:
1992  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1993  Value or fit subtracted from the amplifier image data.
1994  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1995  Value or fit subtracted from the overscan image data.
1996  - ``overscanImage`` : `lsst.afw.image.Image`
1997  Image of the overscan region with the overscan
1998  correction applied. This quantity is used to estimate
1999  the amplifier read noise empirically.
2000 
2001  Raises
2002  ------
2003  RuntimeError
2004  Raised if the ``amp`` does not contain raw pixel information.
2005 
2006  See Also
2007  --------
2008  lsst.ip.isr.isrFunctions.overscanCorrection
2009  """
2010  if amp.getRawHorizontalOverscanBBox().isEmpty():
2011  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
2012  return None
2013 
2014  statControl = afwMath.StatisticsControl()
2015  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2016 
2017  # Determine the bounding boxes
2018  dataBBox = amp.getRawDataBBox()
2019  oscanBBox = amp.getRawHorizontalOverscanBBox()
2020  dx0 = 0
2021  dx1 = 0
2022 
2023  prescanBBox = amp.getRawPrescanBBox()
2024  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
2025  dx0 += self.config.overscanNumLeadingColumnsToSkip
2026  dx1 -= self.config.overscanNumTrailingColumnsToSkip
2027  else:
2028  dx0 += self.config.overscanNumTrailingColumnsToSkip
2029  dx1 -= self.config.overscanNumLeadingColumnsToSkip
2030 
2031  # Determine if we need to work on subregions of the amplifier and overscan.
2032  imageBBoxes = []
2033  overscanBBoxes = []
2034 
2035  if ((self.config.overscanBiasJump
2036  and self.config.overscanBiasJumpLocation)
2037  and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2038  and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
2039  self.config.overscanBiasJumpDevices)):
2040  if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
2041  yLower = self.config.overscanBiasJumpLocation
2042  yUpper = dataBBox.getHeight() - yLower
2043  else:
2044  yUpper = self.config.overscanBiasJumpLocation
2045  yLower = dataBBox.getHeight() - yUpper
2046 
2047  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2048  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
2049  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2050  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2051  yLower)))
2052 
2053  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
2054  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
2055  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
2056  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2057  yUpper)))
2058  else:
2059  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2060  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
2061  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2062  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2063  oscanBBox.getHeight())))
2064 
2065  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
2066  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
2067  ampImage = ccdExposure.maskedImage[imageBBox]
2068  overscanImage = ccdExposure.maskedImage[overscanBBox]
2069 
2070  overscanArray = overscanImage.image.array
2071  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2072  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2073  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
2074 
2075  statControl = afwMath.StatisticsControl()
2076  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2077 
2078  overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp)
2079 
2080  # Measure average overscan levels and record them in the metadata.
2081  levelStat = afwMath.MEDIAN
2082  sigmaStat = afwMath.STDEVCLIP
2083 
2084  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2085  self.config.qa.flatness.nIter)
2086  metadata = ccdExposure.getMetadata()
2087  ampNum = amp.getName()
2088  # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
2089  if isinstance(overscanResults.overscanFit, float):
2090  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2091  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2092  else:
2093  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2094  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2095  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2096 
2097  return overscanResults
2098 
2099  def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None):
2100  """Set the variance plane using the gain and read noise
2101 
2102  The read noise is calculated from the ``overscanImage`` if the
2103  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2104  the value from the amplifier data is used.
2105 
2106  Parameters
2107  ----------
2108  ampExposure : `lsst.afw.image.Exposure`
2109  Exposure to process.
2110  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2111  Amplifier detector data.
2112  overscanImage : `lsst.afw.image.MaskedImage`, optional.
2113  Image of overscan, required only for empirical read noise.
2114  ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
2115  PTC dataset containing the gains and read noise.
2116 
2117 
2118  Raises
2119  ------
2120  RuntimeError
2121  Raised if either ``usePtcGains`` of ``usePtcReadNoise``
2122  are ``True``, but ptcDataset is not provided.
2123 
2124  Raised if ```doEmpiricalReadNoise`` is ``True`` but
2125  ``overscanImage`` is ``None``.
2126 
2127  See also
2128  --------
2129  lsst.ip.isr.isrFunctions.updateVariance
2130  """
2131  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2132  if self.config.usePtcGains:
2133  if ptcDataset is None:
2134  raise RuntimeError("No ptcDataset provided to use PTC gains.")
2135  else:
2136  gain = ptcDataset.gain[amp.getName()]
2137  self.log.info("Using gain from Photon Transfer Curve.")
2138  else:
2139  gain = amp.getGain()
2140 
2141  if math.isnan(gain):
2142  gain = 1.0
2143  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2144  elif gain <= 0:
2145  patchedGain = 1.0
2146  self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
2147  amp.getName(), gain, patchedGain)
2148  gain = patchedGain
2149 
2150  if self.config.doEmpiricalReadNoise and overscanImage is None:
2151  raise RuntimeError("Overscan is none for EmpiricalReadNoise.")
2152 
2153  if self.config.doEmpiricalReadNoise and overscanImage is not None:
2154  stats = afwMath.StatisticsControl()
2155  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2156  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2157  self.log.info("Calculated empirical read noise for amp %s: %f.",
2158  amp.getName(), readNoise)
2159  elif self.config.usePtcReadNoise:
2160  if ptcDataset is None:
2161  raise RuntimeError("No ptcDataset provided to use PTC readnoise.")
2162  else:
2163  readNoise = ptcDataset.noise[amp.getName()]
2164  self.log.info("Using read noise from Photon Transfer Curve.")
2165  else:
2166  readNoise = amp.getReadNoise()
2167 
2168  isrFunctions.updateVariance(
2169  maskedImage=ampExposure.getMaskedImage(),
2170  gain=gain,
2171  readNoise=readNoise,
2172  )
2173 
2174  def darkCorrection(self, exposure, darkExposure, invert=False):
2175  """Apply dark correction in place.
2176 
2177  Parameters
2178  ----------
2179  exposure : `lsst.afw.image.Exposure`
2180  Exposure to process.
2181  darkExposure : `lsst.afw.image.Exposure`
2182  Dark exposure of the same size as ``exposure``.
2183  invert : `Bool`, optional
2184  If True, re-add the dark to an already corrected image.
2185 
2186  Raises
2187  ------
2188  RuntimeError
2189  Raised if either ``exposure`` or ``darkExposure`` do not
2190  have their dark time defined.
2191 
2192  See Also
2193  --------
2194  lsst.ip.isr.isrFunctions.darkCorrection
2195  """
2196  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2197  if math.isnan(expScale):
2198  raise RuntimeError("Exposure darktime is NAN.")
2199  if darkExposure.getInfo().getVisitInfo() is not None \
2200  and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2201  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2202  else:
2203  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
2204  # so getDarkTime() does not exist.
2205  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2206  darkScale = 1.0
2207 
2208  isrFunctions.darkCorrection(
2209  maskedImage=exposure.getMaskedImage(),
2210  darkMaskedImage=darkExposure.getMaskedImage(),
2211  expScale=expScale,
2212  darkScale=darkScale,
2213  invert=invert,
2214  trimToFit=self.config.doTrimToMatchCalib
2215  )
2216 
2217  def doLinearize(self, detector):
2218  """Check if linearization is needed for the detector cameraGeom.
2219 
2220  Checks config.doLinearize and the linearity type of the first
2221  amplifier.
2222 
2223  Parameters
2224  ----------
2225  detector : `lsst.afw.cameraGeom.Detector`
2226  Detector to get linearity type from.
2227 
2228  Returns
2229  -------
2230  doLinearize : `Bool`
2231  If True, linearization should be performed.
2232  """
2233  return self.config.doLinearize and \
2234  detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2235 
2236  def flatCorrection(self, exposure, flatExposure, invert=False):
2237  """Apply flat correction in place.
2238 
2239  Parameters
2240  ----------
2241  exposure : `lsst.afw.image.Exposure`
2242  Exposure to process.
2243  flatExposure : `lsst.afw.image.Exposure`
2244  Flat exposure of the same size as ``exposure``.
2245  invert : `Bool`, optional
2246  If True, unflatten an already flattened image.
2247 
2248  See Also
2249  --------
2250  lsst.ip.isr.isrFunctions.flatCorrection
2251  """
2252  isrFunctions.flatCorrection(
2253  maskedImage=exposure.getMaskedImage(),
2254  flatMaskedImage=flatExposure.getMaskedImage(),
2255  scalingType=self.config.flatScalingType,
2256  userScale=self.config.flatUserScale,
2257  invert=invert,
2258  trimToFit=self.config.doTrimToMatchCalib
2259  )
2260 
2261  def saturationDetection(self, exposure, amp):
2262  """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2263 
2264  Parameters
2265  ----------
2266  exposure : `lsst.afw.image.Exposure`
2267  Exposure to process. Only the amplifier DataSec is processed.
2268  amp : `lsst.afw.table.AmpInfoCatalog`
2269  Amplifier detector data.
2270 
2271  See Also
2272  --------
2273  lsst.ip.isr.isrFunctions.makeThresholdMask
2274  """
2275  if not math.isnan(amp.getSaturation()):
2276  maskedImage = exposure.getMaskedImage()
2277  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2278  isrFunctions.makeThresholdMask(
2279  maskedImage=dataView,
2280  threshold=amp.getSaturation(),
2281  growFootprints=0,
2282  maskName=self.config.saturatedMaskName,
2283  )
2284 
2285  def saturationInterpolation(self, exposure):
2286  """Interpolate over saturated pixels, in place.
2287 
2288  This method should be called after `saturationDetection`, to
2289  ensure that the saturated pixels have been identified in the
2290  SAT mask. It should also be called after `assembleCcd`, since
2291  saturated regions may cross amplifier boundaries.
2292 
2293  Parameters
2294  ----------
2295  exposure : `lsst.afw.image.Exposure`
2296  Exposure to process.
2297 
2298  See Also
2299  --------
2300  lsst.ip.isr.isrTask.saturationDetection
2301  lsst.ip.isr.isrFunctions.interpolateFromMask
2302  """
2303  isrFunctions.interpolateFromMask(
2304  maskedImage=exposure.getMaskedImage(),
2305  fwhm=self.config.fwhm,
2306  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2307  maskNameList=list(self.config.saturatedMaskName),
2308  )
2309 
2310  def suspectDetection(self, exposure, amp):
2311  """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2312 
2313  Parameters
2314  ----------
2315  exposure : `lsst.afw.image.Exposure`
2316  Exposure to process. Only the amplifier DataSec is processed.
2317  amp : `lsst.afw.table.AmpInfoCatalog`
2318  Amplifier detector data.
2319 
2320  See Also
2321  --------
2322  lsst.ip.isr.isrFunctions.makeThresholdMask
2323 
2324  Notes
2325  -----
2326  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2327  This is intended to indicate pixels that may be affected by unknown systematics;
2328  for example if non-linearity corrections above a certain level are unstable
2329  then that would be a useful value for suspectLevel. A value of `nan` indicates
2330  that no such level exists and no pixels are to be masked as suspicious.
2331  """
2332  suspectLevel = amp.getSuspectLevel()
2333  if math.isnan(suspectLevel):
2334  return
2335 
2336  maskedImage = exposure.getMaskedImage()
2337  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2338  isrFunctions.makeThresholdMask(
2339  maskedImage=dataView,
2340  threshold=suspectLevel,
2341  growFootprints=0,
2342  maskName=self.config.suspectMaskName,
2343  )
2344 
2345  def maskDefect(self, exposure, defectBaseList):
2346  """Mask defects using mask plane "BAD", in place.
2347 
2348  Parameters
2349  ----------
2350  exposure : `lsst.afw.image.Exposure`
2351  Exposure to process.
2352  defectBaseList : `lsst.ip.isr.Defects` or `list` of
2353  `lsst.afw.image.DefectBase`.
2354  List of defects to mask.
2355 
2356  Notes
2357  -----
2358  Call this after CCD assembly, since defects may cross amplifier boundaries.
2359  """
2360  maskedImage = exposure.getMaskedImage()
2361  if not isinstance(defectBaseList, Defects):
2362  # Promotes DefectBase to Defect
2363  defectList = Defects(defectBaseList)
2364  else:
2365  defectList = defectBaseList
2366  defectList.maskPixels(maskedImage, maskName="BAD")
2367 
2368  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2369  """Mask edge pixels with applicable mask plane.
2370 
2371  Parameters
2372  ----------
2373  exposure : `lsst.afw.image.Exposure`
2374  Exposure to process.
2375  numEdgePixels : `int`, optional
2376  Number of edge pixels to mask.
2377  maskPlane : `str`, optional
2378  Mask plane name to use.
2379  level : `str`, optional
2380  Level at which to mask edges.
2381  """
2382  maskedImage = exposure.getMaskedImage()
2383  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2384 
2385  if numEdgePixels > 0:
2386  if level == 'DETECTOR':
2387  boxes = [maskedImage.getBBox()]
2388  elif level == 'AMP':
2389  boxes = [amp.getBBox() for amp in exposure.getDetector()]
2390 
2391  for box in boxes:
2392  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2393  subImage = maskedImage[box]
2394  box.grow(-numEdgePixels)
2395  # Mask pixels outside box
2396  SourceDetectionTask.setEdgeBits(
2397  subImage,
2398  box,
2399  maskBitMask)
2400 
2401  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2402  """Mask and interpolate defects using mask plane "BAD", in place.
2403 
2404  Parameters
2405  ----------
2406  exposure : `lsst.afw.image.Exposure`
2407  Exposure to process.
2408  defectBaseList : `lsst.ip.isr.Defects` or `list` of
2409  `lsst.afw.image.DefectBase`.
2410  List of defects to mask and interpolate.
2411 
2412  See Also
2413  --------
2414  lsst.ip.isr.isrTask.maskDefect
2415  """
2416  self.maskDefectmaskDefect(exposure, defectBaseList)
2417  self.maskEdgesmaskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2418  maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2419  isrFunctions.interpolateFromMask(
2420  maskedImage=exposure.getMaskedImage(),
2421  fwhm=self.config.fwhm,
2422  growSaturatedFootprints=0,
2423  maskNameList=["BAD"],
2424  )
2425 
2426  def maskNan(self, exposure):
2427  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2428 
2429  Parameters
2430  ----------
2431  exposure : `lsst.afw.image.Exposure`
2432  Exposure to process.
2433 
2434  Notes
2435  -----
2436  We mask over all non-finite values (NaN, inf), including those
2437  that are masked with other bits (because those may or may not be
2438  interpolated over later, and we want to remove all NaN/infs).
2439  Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2440  preserve the historical name.
2441  """
2442  maskedImage = exposure.getMaskedImage()
2443 
2444  # Find and mask NaNs
2445  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2446  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2447  numNans = maskNans(maskedImage, maskVal)
2448  self.metadata.set("NUMNANS", numNans)
2449  if numNans > 0:
2450  self.log.warn("There were %d unmasked NaNs.", numNans)
2451 
2452  def maskAndInterpolateNan(self, exposure):
2453  """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2454  in place.
2455 
2456  Parameters
2457  ----------
2458  exposure : `lsst.afw.image.Exposure`
2459  Exposure to process.
2460 
2461  See Also
2462  --------
2463  lsst.ip.isr.isrTask.maskNan
2464  """
2465  self.maskNanmaskNan(exposure)
2466  isrFunctions.interpolateFromMask(
2467  maskedImage=exposure.getMaskedImage(),
2468  fwhm=self.config.fwhm,
2469  growSaturatedFootprints=0,
2470  maskNameList=["UNMASKEDNAN"],
2471  )
2472 
2473  def measureBackground(self, exposure, IsrQaConfig=None):
2474  """Measure the image background in subgrids, for quality control purposes.
2475 
2476  Parameters
2477  ----------
2478  exposure : `lsst.afw.image.Exposure`
2479  Exposure to process.
2480  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2481  Configuration object containing parameters on which background
2482  statistics and subgrids to use.
2483  """
2484  if IsrQaConfig is not None:
2485  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2486  IsrQaConfig.flatness.nIter)
2487  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2488  statsControl.setAndMask(maskVal)
2489  maskedImage = exposure.getMaskedImage()
2490  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2491  skyLevel = stats.getValue(afwMath.MEDIAN)
2492  skySigma = stats.getValue(afwMath.STDEVCLIP)
2493  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2494  metadata = exposure.getMetadata()
2495  metadata.set('SKYLEVEL', skyLevel)
2496  metadata.set('SKYSIGMA', skySigma)
2497 
2498  # calcluating flatlevel over the subgrids
2499  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2500  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2501  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2502  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2503  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2504  skyLevels = numpy.zeros((nX, nY))
2505 
2506  for j in range(nY):
2507  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2508  for i in range(nX):
2509  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2510 
2511  xLLC = xc - meshXHalf
2512  yLLC = yc - meshYHalf
2513  xURC = xc + meshXHalf - 1
2514  yURC = yc + meshYHalf - 1
2515 
2516  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2517  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2518 
2519  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2520 
2521  good = numpy.where(numpy.isfinite(skyLevels))
2522  skyMedian = numpy.median(skyLevels[good])
2523  flatness = (skyLevels[good] - skyMedian) / skyMedian
2524  flatness_rms = numpy.std(flatness)
2525  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2526 
2527  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2528  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2529  nX, nY, flatness_pp, flatness_rms)
2530 
2531  metadata.set('FLATNESS_PP', float(flatness_pp))
2532  metadata.set('FLATNESS_RMS', float(flatness_rms))
2533  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2534  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2535  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2536 
2537  def roughZeroPoint(self, exposure):
2538  """Set an approximate magnitude zero point for the exposure.
2539 
2540  Parameters
2541  ----------
2542  exposure : `lsst.afw.image.Exposure`
2543  Exposure to process.
2544  """
2545  filterLabel = exposure.getFilterLabel()
2546  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2547 
2548  if physicalFilter in self.config.fluxMag0T1:
2549  fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2550  else:
2551  self.log.warn("No rough magnitude zero point defined for filter {}.".format(physicalFilter))
2552  fluxMag0 = self.config.defaultFluxMag0T1
2553 
2554  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2555  if not expTime > 0: # handle NaN as well as <= 0
2556  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2557  return
2558 
2559  self.log.info("Setting rough magnitude zero point for filter {}: {}".
2560  format(physicalFilter, 2.5*math.log10(fluxMag0*expTime)))
2561  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2562 
2563  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2564  """Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2565 
2566  Parameters
2567  ----------
2568  ccdExposure : `lsst.afw.image.Exposure`
2569  Exposure to process.
2570  fpPolygon : `lsst.afw.geom.Polygon`
2571  Polygon in focal plane coordinates.
2572  """
2573  # Get ccd corners in focal plane coordinates
2574  ccd = ccdExposure.getDetector()
2575  fpCorners = ccd.getCorners(FOCAL_PLANE)
2576  ccdPolygon = Polygon(fpCorners)
2577 
2578  # Get intersection of ccd corners with fpPolygon
2579  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2580 
2581  # Transform back to pixel positions and build new polygon
2582  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2583  validPolygon = Polygon(ccdPoints)
2584  ccdExposure.getInfo().setValidPolygon(validPolygon)
2585 
2586  @contextmanager
2587  def flatContext(self, exp, flat, dark=None):
2588  """Context manager that applies and removes flats and darks,
2589  if the task is configured to apply them.
2590 
2591  Parameters
2592  ----------
2593  exp : `lsst.afw.image.Exposure`
2594  Exposure to process.
2595  flat : `lsst.afw.image.Exposure`
2596  Flat exposure the same size as ``exp``.
2597  dark : `lsst.afw.image.Exposure`, optional
2598  Dark exposure the same size as ``exp``.
2599 
2600  Yields
2601  ------
2602  exp : `lsst.afw.image.Exposure`
2603  The flat and dark corrected exposure.
2604  """
2605  if self.config.doDark and dark is not None:
2606  self.darkCorrectiondarkCorrection(exp, dark)
2607  if self.config.doFlat:
2608  self.flatCorrectionflatCorrection(exp, flat)
2609  try:
2610  yield exp
2611  finally:
2612  if self.config.doFlat:
2613  self.flatCorrectionflatCorrection(exp, flat, invert=True)
2614  if self.config.doDark and dark is not None:
2615  self.darkCorrectiondarkCorrection(exp, dark, invert=True)
2616 
2617  def debugView(self, exposure, stepname):
2618  """Utility function to examine ISR exposure at different stages.
2619 
2620  Parameters
2621  ----------
2622  exposure : `lsst.afw.image.Exposure`
2623  Exposure to view.
2624  stepname : `str`
2625  State of processing to view.
2626  """
2627  frame = getDebugFrame(self._display, stepname)
2628  if frame:
2629  display = getDisplay(frame)
2630  display.scale('asinh', 'zscale')
2631  display.mtv(exposure)
2632  prompt = "Press Enter to continue [c]... "
2633  while True:
2634  ans = input(prompt).lower()
2635  if ans in ("", "c",):
2636  break
2637 
2638 
2639 class FakeAmp(object):
2640  """A Detector-like object that supports returning gain and saturation level
2641 
2642  This is used when the input exposure does not have a detector.
2643 
2644  Parameters
2645  ----------
2646  exposure : `lsst.afw.image.Exposure`
2647  Exposure to generate a fake amplifier for.
2648  config : `lsst.ip.isr.isrTaskConfig`
2649  Configuration to apply to the fake amplifier.
2650  """
2651 
2652  def __init__(self, exposure, config):
2653  self._bbox_bbox = exposure.getBBox(afwImage.LOCAL)
2654  self._RawHorizontalOverscanBBox_RawHorizontalOverscanBBox = lsst.geom.Box2I()
2655  self._gain_gain = config.gain
2656  self._readNoise_readNoise = config.readNoise
2657  self._saturation_saturation = config.saturation
2658 
2659  def getBBox(self):
2660  return self._bbox_bbox
2661 
2662  def getRawBBox(self):
2663  return self._bbox_bbox
2664 
2666  return self._RawHorizontalOverscanBBox_RawHorizontalOverscanBBox
2667 
2668  def getGain(self):
2669  return self._gain_gain
2670 
2671  def getReadNoise(self):
2672  return self._readNoise_readNoise
2673 
2674  def getSaturation(self):
2675  return self._saturation_saturation
2676 
2677  def getSuspectLevel(self):
2678  return float("NaN")
2679 
2680 
2681 class RunIsrConfig(pexConfig.Config):
2682  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2683 
2684 
2685 class RunIsrTask(pipeBase.CmdLineTask):
2686  """Task to wrap the default IsrTask to allow it to be retargeted.
2687 
2688  The standard IsrTask can be called directly from a command line
2689  program, but doing so removes the ability of the task to be
2690  retargeted. As most cameras override some set of the IsrTask
2691  methods, this would remove those data-specific methods in the
2692  output post-ISR images. This wrapping class fixes the issue,
2693  allowing identical post-ISR images to be generated by both the
2694  processCcd and isrTask code.
2695  """
2696  ConfigClass = RunIsrConfig
2697  _DefaultName = "runIsr"
2698 
2699  def __init__(self, *args, **kwargs):
2700  super().__init__(*args, **kwargs)
2701  self.makeSubtask("isr")
2702 
2703  def runDataRef(self, dataRef):
2704  """
2705  Parameters
2706  ----------
2707  dataRef : `lsst.daf.persistence.ButlerDataRef`
2708  data reference of the detector data to be processed
2709 
2710  Returns
2711  -------
2712  result : `pipeBase.Struct`
2713  Result struct with component:
2714 
2715  - exposure : `lsst.afw.image.Exposure`
2716  Post-ISR processed exposure.
2717  """
2718  return self.isr.runDataRef(dataRef)
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2665
def __init__(self, exposure, config)
Definition: isrTask.py:2652
def __init__(self, *config=None)
Definition: isrTask.py:265
def flatCorrection(self, exposure, flatExposure, invert=False)
Definition: isrTask.py:2236
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2452
def saturationInterpolation(self, exposure)
Definition: isrTask.py:2285
def runDataRef(self, sensorRef)
Definition: isrTask.py:1716
def maskNan(self, exposure)
Definition: isrTask.py:2426
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1900
def debugView(self, exposure, stepname)
Definition: isrTask.py:2617
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Definition: isrTask.py:1767
def saturationDetection(self, exposure, amp)
Definition: isrTask.py:2261
def maskDefect(self, exposure, defectBaseList)
Definition: isrTask.py:2345
def __init__(self, **kwargs)
Definition: isrTask.py:937
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:947
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
Definition: isrTask.py:2368
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1971
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2473
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2537
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2401
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Definition: isrTask.py:2563
def readIsrData(self, dataRef, rawExposure)
Definition: isrTask.py:1044
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1815
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Definition: isrTask.py:1226
def doLinearize(self, detector)
Definition: isrTask.py:2217
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2587
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1863
def suspectDetection(self, exposure, amp)
Definition: isrTask.py:2310
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
Definition: isrTask.py:2099
def darkCorrection(self, exposure, darkExposure, invert=False)
Definition: isrTask.py:2174
def __init__(self, *args, **kwargs)
Definition: isrTask.py:2699
def runDataRef(self, dataRef)
Definition: isrTask.py:2703
def checkFilter(exposure, filterList, log)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
Definition: isrTask.py:62
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35