lsst.ip.isr  21.0.0-6-g4e60332+b9160e71e7
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36  ReadoutCorner)
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 
43 from . import isrFunctions
44 from . import isrQa
45 from . import linearize
46 from .defects import Defects
47 
48 from .assembleCcdTask import AssembleCcdTask
49 from .crosstalk import CrosstalkTask, CrosstalkCalib
50 from .fringe import FringeTask
51 from .isr import maskNans
52 from .masking import MaskingTask
53 from .overscan import OverscanCorrectionTask
54 from .straylight import StrayLightTask
55 from .vignette import VignetteTask
56 from lsst.daf.butler import DimensionGraph
57 
58 
59 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
60 
61 
62 def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections):
63  """Lookup function to identify crosstalkSource entries.
64 
65  This should return an empty list under most circumstances. Only
66  when inter-chip crosstalk has been identified should this be
67  populated.
68 
69  This will be unused until DM-25348 resolves the quantum graph
70  generation issue.
71 
72  Parameters
73  ----------
74  datasetType : `str`
75  Dataset to lookup.
76  registry : `lsst.daf.butler.Registry`
77  Butler registry to query.
78  quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79  Data id to transform to identify crosstalkSources. The
80  ``detector`` entry will be stripped.
81  collections : `lsst.daf.butler.CollectionSearch`
82  Collections to search through.
83 
84  Returns
85  -------
86  results : `list` [`lsst.daf.butler.DatasetRef`]
87  List of datasets that match the query that will be used as
88  crosstalkSources.
89  """
90  newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"]))
91  results = list(registry.queryDatasets(datasetType,
92  collections=collections,
93  dataId=newDataId,
94  findFirst=True,
95  ).expanded())
96  return results
97 
98 
99 class IsrTaskConnections(pipeBase.PipelineTaskConnections,
100  dimensions={"instrument", "exposure", "detector"},
101  defaultTemplates={}):
102  ccdExposure = cT.Input(
103  name="raw",
104  doc="Input exposure to process.",
105  storageClass="Exposure",
106  dimensions=["instrument", "exposure", "detector"],
107  )
108  camera = cT.PrerequisiteInput(
109  name="camera",
110  storageClass="Camera",
111  doc="Input camera to construct complete exposures.",
112  dimensions=["instrument"],
113  isCalibration=True,
114  )
115 
116  crosstalk = cT.PrerequisiteInput(
117  name="crosstalk",
118  doc="Input crosstalk object",
119  storageClass="CrosstalkCalib",
120  dimensions=["instrument", "detector"],
121  isCalibration=True,
122  )
123  # TODO: DM-25348. This does not work yet to correctly load
124  # possible crosstalk sources.
125  crosstalkSources = cT.PrerequisiteInput(
126  name="isrOverscanCorrected",
127  doc="Overscan corrected input images.",
128  storageClass="Exposure",
129  dimensions=["instrument", "exposure", "detector"],
130  deferLoad=True,
131  multiple=True,
132  lookupFunction=crosstalkSourceLookup,
133  )
134  bias = cT.PrerequisiteInput(
135  name="bias",
136  doc="Input bias calibration.",
137  storageClass="ExposureF",
138  dimensions=["instrument", "detector"],
139  isCalibration=True,
140  )
141  dark = cT.PrerequisiteInput(
142  name='dark',
143  doc="Input dark calibration.",
144  storageClass="ExposureF",
145  dimensions=["instrument", "detector"],
146  isCalibration=True,
147  )
148  flat = cT.PrerequisiteInput(
149  name="flat",
150  doc="Input flat calibration.",
151  storageClass="ExposureF",
152  dimensions=["instrument", "physical_filter", "detector"],
153  isCalibration=True,
154  )
155  fringes = cT.PrerequisiteInput(
156  name="fringe",
157  doc="Input fringe calibration.",
158  storageClass="ExposureF",
159  dimensions=["instrument", "physical_filter", "detector"],
160  isCalibration=True,
161  )
162  strayLightData = cT.PrerequisiteInput(
163  name='yBackground',
164  doc="Input stray light calibration.",
165  storageClass="StrayLightData",
166  dimensions=["instrument", "physical_filter", "detector"],
167  isCalibration=True,
168  )
169  bfKernel = cT.PrerequisiteInput(
170  name='bfKernel',
171  doc="Input brighter-fatter kernel.",
172  storageClass="NumpyArray",
173  dimensions=["instrument"],
174  isCalibration=True,
175  )
176  newBFKernel = cT.PrerequisiteInput(
177  name='brighterFatterKernel',
178  doc="Newer complete kernel + gain solutions.",
179  storageClass="BrighterFatterKernel",
180  dimensions=["instrument", "detector"],
181  isCalibration=True,
182  )
183  defects = cT.PrerequisiteInput(
184  name='defects',
185  doc="Input defect tables.",
186  storageClass="Defects",
187  dimensions=["instrument", "detector"],
188  isCalibration=True,
189  )
190  opticsTransmission = cT.PrerequisiteInput(
191  name="transmission_optics",
192  storageClass="TransmissionCurve",
193  doc="Transmission curve due to the optics.",
194  dimensions=["instrument"],
195  isCalibration=True,
196  )
197  filterTransmission = cT.PrerequisiteInput(
198  name="transmission_filter",
199  storageClass="TransmissionCurve",
200  doc="Transmission curve due to the filter.",
201  dimensions=["instrument", "physical_filter"],
202  isCalibration=True,
203  )
204  sensorTransmission = cT.PrerequisiteInput(
205  name="transmission_sensor",
206  storageClass="TransmissionCurve",
207  doc="Transmission curve due to the sensor.",
208  dimensions=["instrument", "detector"],
209  isCalibration=True,
210  )
211  atmosphereTransmission = cT.PrerequisiteInput(
212  name="transmission_atmosphere",
213  storageClass="TransmissionCurve",
214  doc="Transmission curve due to the atmosphere.",
215  dimensions=["instrument"],
216  isCalibration=True,
217  )
218  illumMaskedImage = cT.PrerequisiteInput(
219  name="illum",
220  doc="Input illumination correction.",
221  storageClass="MaskedImageF",
222  dimensions=["instrument", "physical_filter", "detector"],
223  isCalibration=True,
224  )
225 
226  outputExposure = cT.Output(
227  name='postISRCCD',
228  doc="Output ISR processed exposure.",
229  storageClass="Exposure",
230  dimensions=["instrument", "exposure", "detector"],
231  )
232  preInterpExposure = cT.Output(
233  name='preInterpISRCCD',
234  doc="Output ISR processed exposure, with pixels left uninterpolated.",
235  storageClass="ExposureF",
236  dimensions=["instrument", "exposure", "detector"],
237  )
238  outputOssThumbnail = cT.Output(
239  name="OssThumb",
240  doc="Output Overscan-subtracted thumbnail image.",
241  storageClass="Thumbnail",
242  dimensions=["instrument", "exposure", "detector"],
243  )
244  outputFlattenedThumbnail = cT.Output(
245  name="FlattenedThumb",
246  doc="Output flat-corrected thumbnail image.",
247  storageClass="Thumbnail",
248  dimensions=["instrument", "exposure", "detector"],
249  )
250 
251  def __init__(self, *, config=None):
252  super().__init__(config=config)
253 
254  if config.doBias is not True:
255  self.prerequisiteInputs.discard("bias")
256  if config.doLinearize is not True:
257  self.prerequisiteInputs.discard("linearizer")
258  if config.doCrosstalk is not True:
259  self.inputs.discard("crosstalkSources")
260  self.prerequisiteInputs.discard("crosstalk")
261  if config.doBrighterFatter is not True:
262  self.prerequisiteInputs.discard("bfKernel")
263  self.prerequisiteInputs.discard("newBFKernel")
264  if config.doDefect is not True:
265  self.prerequisiteInputs.discard("defects")
266  if config.doDark is not True:
267  self.prerequisiteInputs.discard("dark")
268  if config.doFlat is not True:
269  self.prerequisiteInputs.discard("flat")
270  if config.doAttachTransmissionCurve is not True:
271  self.prerequisiteInputs.discard("opticsTransmission")
272  self.prerequisiteInputs.discard("filterTransmission")
273  self.prerequisiteInputs.discard("sensorTransmission")
274  self.prerequisiteInputs.discard("atmosphereTransmission")
275  if config.doUseOpticsTransmission is not True:
276  self.prerequisiteInputs.discard("opticsTransmission")
277  if config.doUseFilterTransmission is not True:
278  self.prerequisiteInputs.discard("filterTransmission")
279  if config.doUseSensorTransmission is not True:
280  self.prerequisiteInputs.discard("sensorTransmission")
281  if config.doUseAtmosphereTransmission is not True:
282  self.prerequisiteInputs.discard("atmosphereTransmission")
283  if config.doIlluminationCorrection is not True:
284  self.prerequisiteInputs.discard("illumMaskedImage")
285 
286  if config.doWrite is not True:
287  self.outputs.discard("outputExposure")
288  self.outputs.discard("preInterpExposure")
289  self.outputs.discard("outputFlattenedThumbnail")
290  self.outputs.discard("outputOssThumbnail")
291  if config.doSaveInterpPixels is not True:
292  self.outputs.discard("preInterpExposure")
293  if config.qa.doThumbnailOss is not True:
294  self.outputs.discard("outputOssThumbnail")
295  if config.qa.doThumbnailFlattened is not True:
296  self.outputs.discard("outputFlattenedThumbnail")
297 
298 
299 class IsrTaskConfig(pipeBase.PipelineTaskConfig,
300  pipelineConnections=IsrTaskConnections):
301  """Configuration parameters for IsrTask.
302 
303  Items are grouped in the order in which they are executed by the task.
304  """
305  datasetType = pexConfig.Field(
306  dtype=str,
307  doc="Dataset type for input data; users will typically leave this alone, "
308  "but camera-specific ISR tasks will override it",
309  default="raw",
310  )
311 
312  fallbackFilterName = pexConfig.Field(
313  dtype=str,
314  doc="Fallback default filter name for calibrations.",
315  optional=True
316  )
317  useFallbackDate = pexConfig.Field(
318  dtype=bool,
319  doc="Pass observation date when using fallback filter.",
320  default=False,
321  )
322  expectWcs = pexConfig.Field(
323  dtype=bool,
324  default=True,
325  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
326  )
327  fwhm = pexConfig.Field(
328  dtype=float,
329  doc="FWHM of PSF in arcseconds.",
330  default=1.0,
331  )
332  qa = pexConfig.ConfigField(
333  dtype=isrQa.IsrQaConfig,
334  doc="QA related configuration options.",
335  )
336 
337  # Image conversion configuration
338  doConvertIntToFloat = pexConfig.Field(
339  dtype=bool,
340  doc="Convert integer raw images to floating point values?",
341  default=True,
342  )
343 
344  # Saturated pixel handling.
345  doSaturation = pexConfig.Field(
346  dtype=bool,
347  doc="Mask saturated pixels? NB: this is totally independent of the"
348  " interpolation option - this is ONLY setting the bits in the mask."
349  " To have them interpolated make sure doSaturationInterpolation=True",
350  default=True,
351  )
352  saturatedMaskName = pexConfig.Field(
353  dtype=str,
354  doc="Name of mask plane to use in saturation detection and interpolation",
355  default="SAT",
356  )
357  saturation = pexConfig.Field(
358  dtype=float,
359  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
360  default=float("NaN"),
361  )
362  growSaturationFootprintSize = pexConfig.Field(
363  dtype=int,
364  doc="Number of pixels by which to grow the saturation footprints",
365  default=1,
366  )
367 
368  # Suspect pixel handling.
369  doSuspect = pexConfig.Field(
370  dtype=bool,
371  doc="Mask suspect pixels?",
372  default=False,
373  )
374  suspectMaskName = pexConfig.Field(
375  dtype=str,
376  doc="Name of mask plane to use for suspect pixels",
377  default="SUSPECT",
378  )
379  numEdgeSuspect = pexConfig.Field(
380  dtype=int,
381  doc="Number of edge pixels to be flagged as untrustworthy.",
382  default=0,
383  )
384  edgeMaskLevel = pexConfig.ChoiceField(
385  dtype=str,
386  doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
387  default="DETECTOR",
388  allowed={
389  'DETECTOR': 'Mask only the edges of the full detector.',
390  'AMP': 'Mask edges of each amplifier.',
391  },
392  )
393 
394  # Initial masking options.
395  doSetBadRegions = pexConfig.Field(
396  dtype=bool,
397  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
398  default=True,
399  )
400  badStatistic = pexConfig.ChoiceField(
401  dtype=str,
402  doc="How to estimate the average value for BAD regions.",
403  default='MEANCLIP',
404  allowed={
405  "MEANCLIP": "Correct using the (clipped) mean of good data",
406  "MEDIAN": "Correct using the median of the good data",
407  },
408  )
409 
410  # Overscan subtraction configuration.
411  doOverscan = pexConfig.Field(
412  dtype=bool,
413  doc="Do overscan subtraction?",
414  default=True,
415  )
416  overscan = pexConfig.ConfigurableField(
417  target=OverscanCorrectionTask,
418  doc="Overscan subtraction task for image segments.",
419  )
420 
421  overscanFitType = pexConfig.ChoiceField(
422  dtype=str,
423  doc="The method for fitting the overscan bias level.",
424  default='MEDIAN',
425  allowed={
426  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
427  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
428  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
429  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
430  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
431  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
432  "MEAN": "Correct using the mean of the overscan region",
433  "MEANCLIP": "Correct using a clipped mean of the overscan region",
434  "MEDIAN": "Correct using the median of the overscan region",
435  "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
436  },
437  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
438  " This option will no longer be used, and will be removed after v20.")
439  )
440  overscanOrder = pexConfig.Field(
441  dtype=int,
442  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, "
443  "or number of spline knots if overscan fit type is a spline."),
444  default=1,
445  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
446  " This option will no longer be used, and will be removed after v20.")
447  )
448  overscanNumSigmaClip = pexConfig.Field(
449  dtype=float,
450  doc="Rejection threshold (sigma) for collapsing overscan before fit",
451  default=3.0,
452  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
453  " This option will no longer be used, and will be removed after v20.")
454  )
455  overscanIsInt = pexConfig.Field(
456  dtype=bool,
457  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
458  " and overscan.FitType=MEDIAN_PER_ROW.",
459  default=True,
460  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
461  " This option will no longer be used, and will be removed after v20.")
462  )
463  # These options do not get deprecated, as they define how we slice up the image data.
464  overscanNumLeadingColumnsToSkip = pexConfig.Field(
465  dtype=int,
466  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
467  default=0,
468  )
469  overscanNumTrailingColumnsToSkip = pexConfig.Field(
470  dtype=int,
471  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
472  default=0,
473  )
474  overscanMaxDev = pexConfig.Field(
475  dtype=float,
476  doc="Maximum deviation from the median for overscan",
477  default=1000.0, check=lambda x: x > 0
478  )
479  overscanBiasJump = pexConfig.Field(
480  dtype=bool,
481  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
482  default=False,
483  )
484  overscanBiasJumpKeyword = pexConfig.Field(
485  dtype=str,
486  doc="Header keyword containing information about devices.",
487  default="NO_SUCH_KEY",
488  )
489  overscanBiasJumpDevices = pexConfig.ListField(
490  dtype=str,
491  doc="List of devices that need piecewise overscan correction.",
492  default=(),
493  )
494  overscanBiasJumpLocation = pexConfig.Field(
495  dtype=int,
496  doc="Location of bias jump along y-axis.",
497  default=0,
498  )
499 
500  # Amplifier to CCD assembly configuration
501  doAssembleCcd = pexConfig.Field(
502  dtype=bool,
503  default=True,
504  doc="Assemble amp-level exposures into a ccd-level exposure?"
505  )
506  assembleCcd = pexConfig.ConfigurableField(
507  target=AssembleCcdTask,
508  doc="CCD assembly task",
509  )
510 
511  # General calibration configuration.
512  doAssembleIsrExposures = pexConfig.Field(
513  dtype=bool,
514  default=False,
515  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
516  )
517  doTrimToMatchCalib = pexConfig.Field(
518  dtype=bool,
519  default=False,
520  doc="Trim raw data to match calibration bounding boxes?"
521  )
522 
523  # Bias subtraction.
524  doBias = pexConfig.Field(
525  dtype=bool,
526  doc="Apply bias frame correction?",
527  default=True,
528  )
529  biasDataProductName = pexConfig.Field(
530  dtype=str,
531  doc="Name of the bias data product",
532  default="bias",
533  )
534  doBiasBeforeOverscan = pexConfig.Field(
535  dtype=bool,
536  doc="Reverse order of overscan and bias correction.",
537  default=False
538  )
539 
540  # Variance construction
541  doVariance = pexConfig.Field(
542  dtype=bool,
543  doc="Calculate variance?",
544  default=True
545  )
546  gain = pexConfig.Field(
547  dtype=float,
548  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
549  default=float("NaN"),
550  )
551  readNoise = pexConfig.Field(
552  dtype=float,
553  doc="The read noise to use if no Detector is present in the Exposure",
554  default=0.0,
555  )
556  doEmpiricalReadNoise = pexConfig.Field(
557  dtype=bool,
558  default=False,
559  doc="Calculate empirical read noise instead of value from AmpInfo data?"
560  )
561 
562  # Linearization.
563  doLinearize = pexConfig.Field(
564  dtype=bool,
565  doc="Correct for nonlinearity of the detector's response?",
566  default=True,
567  )
568 
569  # Crosstalk.
570  doCrosstalk = pexConfig.Field(
571  dtype=bool,
572  doc="Apply intra-CCD crosstalk correction?",
573  default=False,
574  )
575  doCrosstalkBeforeAssemble = pexConfig.Field(
576  dtype=bool,
577  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
578  default=False,
579  )
580  crosstalk = pexConfig.ConfigurableField(
581  target=CrosstalkTask,
582  doc="Intra-CCD crosstalk correction",
583  )
584 
585  # Masking options.
586  doDefect = pexConfig.Field(
587  dtype=bool,
588  doc="Apply correction for CCD defects, e.g. hot pixels?",
589  default=True,
590  )
591  doNanMasking = pexConfig.Field(
592  dtype=bool,
593  doc="Mask NAN pixels?",
594  default=True,
595  )
596  doWidenSaturationTrails = pexConfig.Field(
597  dtype=bool,
598  doc="Widen bleed trails based on their width?",
599  default=True
600  )
601 
602  # Brighter-Fatter correction.
603  doBrighterFatter = pexConfig.Field(
604  dtype=bool,
605  default=False,
606  doc="Apply the brighter fatter correction"
607  )
608  brighterFatterLevel = pexConfig.ChoiceField(
609  dtype=str,
610  default="DETECTOR",
611  doc="The level at which to correct for brighter-fatter.",
612  allowed={
613  "AMP": "Every amplifier treated separately.",
614  "DETECTOR": "One kernel per detector",
615  }
616  )
617  brighterFatterMaxIter = pexConfig.Field(
618  dtype=int,
619  default=10,
620  doc="Maximum number of iterations for the brighter fatter correction"
621  )
622  brighterFatterThreshold = pexConfig.Field(
623  dtype=float,
624  default=1000,
625  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
626  " absolute value of the difference between the current corrected image and the one"
627  " from the previous iteration summed over all the pixels."
628  )
629  brighterFatterApplyGain = pexConfig.Field(
630  dtype=bool,
631  default=True,
632  doc="Should the gain be applied when applying the brighter fatter correction?"
633  )
634  brighterFatterMaskGrowSize = pexConfig.Field(
635  dtype=int,
636  default=0,
637  doc="Number of pixels to grow the masks listed in config.maskListToInterpolate "
638  " when brighter-fatter correction is applied."
639  )
640 
641  # Dark subtraction.
642  doDark = pexConfig.Field(
643  dtype=bool,
644  doc="Apply dark frame correction?",
645  default=True,
646  )
647  darkDataProductName = pexConfig.Field(
648  dtype=str,
649  doc="Name of the dark data product",
650  default="dark",
651  )
652 
653  # Camera-specific stray light removal.
654  doStrayLight = pexConfig.Field(
655  dtype=bool,
656  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
657  default=False,
658  )
659  strayLight = pexConfig.ConfigurableField(
660  target=StrayLightTask,
661  doc="y-band stray light correction"
662  )
663 
664  # Flat correction.
665  doFlat = pexConfig.Field(
666  dtype=bool,
667  doc="Apply flat field correction?",
668  default=True,
669  )
670  flatDataProductName = pexConfig.Field(
671  dtype=str,
672  doc="Name of the flat data product",
673  default="flat",
674  )
675  flatScalingType = pexConfig.ChoiceField(
676  dtype=str,
677  doc="The method for scaling the flat on the fly.",
678  default='USER',
679  allowed={
680  "USER": "Scale by flatUserScale",
681  "MEAN": "Scale by the inverse of the mean",
682  "MEDIAN": "Scale by the inverse of the median",
683  },
684  )
685  flatUserScale = pexConfig.Field(
686  dtype=float,
687  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
688  default=1.0,
689  )
690  doTweakFlat = pexConfig.Field(
691  dtype=bool,
692  doc="Tweak flats to match observed amplifier ratios?",
693  default=False
694  )
695 
696  # Amplifier normalization based on gains instead of using flats configuration.
697  doApplyGains = pexConfig.Field(
698  dtype=bool,
699  doc="Correct the amplifiers for their gains instead of applying flat correction",
700  default=False,
701  )
702  normalizeGains = pexConfig.Field(
703  dtype=bool,
704  doc="Normalize all the amplifiers in each CCD to have the same median value.",
705  default=False,
706  )
707 
708  # Fringe correction.
709  doFringe = pexConfig.Field(
710  dtype=bool,
711  doc="Apply fringe correction?",
712  default=True,
713  )
714  fringe = pexConfig.ConfigurableField(
715  target=FringeTask,
716  doc="Fringe subtraction task",
717  )
718  fringeAfterFlat = pexConfig.Field(
719  dtype=bool,
720  doc="Do fringe subtraction after flat-fielding?",
721  default=True,
722  )
723 
724  # Initial CCD-level background statistics options.
725  doMeasureBackground = pexConfig.Field(
726  dtype=bool,
727  doc="Measure the background level on the reduced image?",
728  default=False,
729  )
730 
731  # Camera-specific masking configuration.
732  doCameraSpecificMasking = pexConfig.Field(
733  dtype=bool,
734  doc="Mask camera-specific bad regions?",
735  default=False,
736  )
737  masking = pexConfig.ConfigurableField(
738  target=MaskingTask,
739  doc="Masking task."
740  )
741 
742  # Interpolation options.
743 
744  doInterpolate = pexConfig.Field(
745  dtype=bool,
746  doc="Interpolate masked pixels?",
747  default=True,
748  )
749  doSaturationInterpolation = pexConfig.Field(
750  dtype=bool,
751  doc="Perform interpolation over pixels masked as saturated?"
752  " NB: This is independent of doSaturation; if that is False this plane"
753  " will likely be blank, resulting in a no-op here.",
754  default=True,
755  )
756  doNanInterpolation = pexConfig.Field(
757  dtype=bool,
758  doc="Perform interpolation over pixels masked as NaN?"
759  " NB: This is independent of doNanMasking; if that is False this plane"
760  " will likely be blank, resulting in a no-op here.",
761  default=True,
762  )
763  doNanInterpAfterFlat = pexConfig.Field(
764  dtype=bool,
765  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
766  "also have to interpolate them before flat-fielding."),
767  default=False,
768  )
769  maskListToInterpolate = pexConfig.ListField(
770  dtype=str,
771  doc="List of mask planes that should be interpolated.",
772  default=['SAT', 'BAD', 'UNMASKEDNAN'],
773  )
774  doSaveInterpPixels = pexConfig.Field(
775  dtype=bool,
776  doc="Save a copy of the pre-interpolated pixel values?",
777  default=False,
778  )
779 
780  # Default photometric calibration options.
781  fluxMag0T1 = pexConfig.DictField(
782  keytype=str,
783  itemtype=float,
784  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
785  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
786  ))
787  )
788  defaultFluxMag0T1 = pexConfig.Field(
789  dtype=float,
790  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
791  default=pow(10.0, 0.4*28.0)
792  )
793 
794  # Vignette correction configuration.
795  doVignette = pexConfig.Field(
796  dtype=bool,
797  doc="Apply vignetting parameters?",
798  default=False,
799  )
800  vignette = pexConfig.ConfigurableField(
801  target=VignetteTask,
802  doc="Vignetting task.",
803  )
804 
805  # Transmission curve configuration.
806  doAttachTransmissionCurve = pexConfig.Field(
807  dtype=bool,
808  default=False,
809  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
810  )
811  doUseOpticsTransmission = pexConfig.Field(
812  dtype=bool,
813  default=True,
814  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
815  )
816  doUseFilterTransmission = pexConfig.Field(
817  dtype=bool,
818  default=True,
819  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
820  )
821  doUseSensorTransmission = pexConfig.Field(
822  dtype=bool,
823  default=True,
824  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
825  )
826  doUseAtmosphereTransmission = pexConfig.Field(
827  dtype=bool,
828  default=True,
829  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
830  )
831 
832  # Illumination correction.
833  doIlluminationCorrection = pexConfig.Field(
834  dtype=bool,
835  default=False,
836  doc="Perform illumination correction?"
837  )
838  illuminationCorrectionDataProductName = pexConfig.Field(
839  dtype=str,
840  doc="Name of the illumination correction data product.",
841  default="illumcor",
842  )
843  illumScale = pexConfig.Field(
844  dtype=float,
845  doc="Scale factor for the illumination correction.",
846  default=1.0,
847  )
848  illumFilters = pexConfig.ListField(
849  dtype=str,
850  default=[],
851  doc="Only perform illumination correction for these filters."
852  )
853 
854  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
855  doWrite = pexConfig.Field(
856  dtype=bool,
857  doc="Persist postISRCCD?",
858  default=True,
859  )
860 
861  def validate(self):
862  super().validate()
863  if self.doFlatdoFlat and self.doApplyGainsdoApplyGains:
864  raise ValueError("You may not specify both doFlat and doApplyGains")
865  if self.doBiasBeforeOverscandoBiasBeforeOverscan and self.doTrimToMatchCalibdoTrimToMatchCalib:
866  raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
867  if self.doSaturationInterpolationdoSaturationInterpolation and self.saturatedMaskNamesaturatedMaskName not in self.maskListToInterpolatemaskListToInterpolate:
868  self.maskListToInterpolatemaskListToInterpolate.append(self.saturatedMaskNamesaturatedMaskName)
869  if not self.doSaturationInterpolationdoSaturationInterpolation and self.saturatedMaskNamesaturatedMaskName in self.maskListToInterpolatemaskListToInterpolate:
870  self.maskListToInterpolatemaskListToInterpolate.remove(self.saturatedMaskNamesaturatedMaskName)
871  if self.doNanInterpolationdoNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolatemaskListToInterpolate:
872  self.maskListToInterpolatemaskListToInterpolate.append("UNMASKEDNAN")
873 
874 
875 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
876  """Apply common instrument signature correction algorithms to a raw frame.
877 
878  The process for correcting imaging data is very similar from
879  camera to camera. This task provides a vanilla implementation of
880  doing these corrections, including the ability to turn certain
881  corrections off if they are not needed. The inputs to the primary
882  method, `run()`, are a raw exposure to be corrected and the
883  calibration data products. The raw input is a single chip sized
884  mosaic of all amps including overscans and other non-science
885  pixels. The method `runDataRef()` identifies and defines the
886  calibration data products, and is intended for use by a
887  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
888  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
889  subclassed for different camera, although the most camera specific
890  methods have been split into subtasks that can be redirected
891  appropriately.
892 
893  The __init__ method sets up the subtasks for ISR processing, using
894  the defaults from `lsst.ip.isr`.
895 
896  Parameters
897  ----------
898  args : `list`
899  Positional arguments passed to the Task constructor. None used at this time.
900  kwargs : `dict`, optional
901  Keyword arguments passed on to the Task constructor. None used at this time.
902  """
903  ConfigClass = IsrTaskConfig
904  _DefaultName = "isr"
905 
906  def __init__(self, **kwargs):
907  super().__init__(**kwargs)
908  self.makeSubtask("assembleCcd")
909  self.makeSubtask("crosstalk")
910  self.makeSubtask("strayLight")
911  self.makeSubtask("fringe")
912  self.makeSubtask("masking")
913  self.makeSubtask("overscan")
914  self.makeSubtask("vignette")
915 
916  def runQuantum(self, butlerQC, inputRefs, outputRefs):
917  inputs = butlerQC.get(inputRefs)
918 
919  try:
920  inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
921  except Exception as e:
922  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
923  (inputRefs, e))
924 
925  inputs['isGen3'] = True
926 
927  detector = inputs['ccdExposure'].getDetector()
928 
929  if self.config.doCrosstalk is True:
930  # Crosstalk sources need to be defined by the pipeline
931  # yaml if they exist.
932  if 'crosstalk' in inputs and inputs['crosstalk'] is not None:
933  if not isinstance(inputs['crosstalk'], CrosstalkCalib):
934  inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk'])
935  else:
936  coeffVector = (self.config.crosstalk.crosstalkValues
937  if self.config.crosstalk.useConfigCoefficients else None)
938  crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
939  inputs['crosstalk'] = crosstalkCalib
940  if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0:
941  if 'crosstalkSources' not in inputs:
942  self.log.warn("No crosstalkSources found for chip with interChip terms!")
943 
944  if self.doLinearizedoLinearize(detector) is True:
945  if 'linearizer' in inputs and isinstance(inputs['linearizer'], dict):
946  linearizer = linearize.Linearizer(detector=detector, log=self.log)
947  linearizer.fromYaml(inputs['linearizer'])
948  else:
949  linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), detector=detector,
950  log=self.log)
951  inputs['linearizer'] = linearizer
952 
953  if self.config.doDefect is True:
954  if "defects" in inputs and inputs['defects'] is not None:
955  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
956  # masking expects a list of defects defined by their bounding box
957  if not isinstance(inputs["defects"], Defects):
958  inputs["defects"] = Defects.fromTable(inputs["defects"])
959 
960  # Load the correct style of brighter fatter kernel, and repack
961  # the information as a numpy array.
962  if self.config.doBrighterFatter:
963  brighterFatterKernel = inputs.pop('newBFKernel', None)
964  if brighterFatterKernel is None:
965  brighterFatterKernel = inputs.get('bfKernel', None)
966 
967  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
968  detId = detector.getId()
969  inputs['bfGains'] = brighterFatterKernel.gain
970  # If the kernel is not an ndarray, it's the cp_pipe version
971  # so extract the kernel for this detector, or raise an error
972  if self.config.brighterFatterLevel == 'DETECTOR':
973  if brighterFatterKernel.detectorKernel:
974  inputs['bfKernel'] = brighterFatterKernel.detectorKernel[detId]
975  elif brighterFatterKernel.detectorKernelFromAmpKernels:
976  inputs['bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
977  else:
978  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
979  else:
980  # TODO DM-15631 for implementing this
981  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
982 
983  if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
984  expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
985  inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
986  expId=expId,
987  assembler=self.assembleCcd
988  if self.config.doAssembleIsrExposures else None)
989  else:
990  inputs['fringes'] = pipeBase.Struct(fringes=None)
991 
992  if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
993  if 'strayLightData' not in inputs:
994  inputs['strayLightData'] = None
995 
996  outputs = self.runrun(**inputs)
997  butlerQC.put(outputs, outputRefs)
998 
999  def readIsrData(self, dataRef, rawExposure):
1000  """Retrieve necessary frames for instrument signature removal.
1001 
1002  Pre-fetching all required ISR data products limits the IO
1003  required by the ISR. Any conflict between the calibration data
1004  available and that needed for ISR is also detected prior to
1005  doing processing, allowing it to fail quickly.
1006 
1007  Parameters
1008  ----------
1009  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1010  Butler reference of the detector data to be processed
1011  rawExposure : `afw.image.Exposure`
1012  The raw exposure that will later be corrected with the
1013  retrieved calibration data; should not be modified in this
1014  method.
1015 
1016  Returns
1017  -------
1018  result : `lsst.pipe.base.Struct`
1019  Result struct with components (which may be `None`):
1020  - ``bias``: bias calibration frame (`afw.image.Exposure`)
1021  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
1022  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1023  - ``dark``: dark calibration frame (`afw.image.Exposure`)
1024  - ``flat``: flat calibration frame (`afw.image.Exposure`)
1025  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1026  - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1027  - ``fringes``: `lsst.pipe.base.Struct` with components:
1028  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1029  - ``seed``: random seed derived from the ccdExposureId for random
1030  number generator (`uint32`).
1031  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1032  A ``TransmissionCurve`` that represents the throughput of the optics,
1033  to be evaluated in focal-plane coordinates.
1034  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1035  A ``TransmissionCurve`` that represents the throughput of the filter
1036  itself, to be evaluated in focal-plane coordinates.
1037  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1038  A ``TransmissionCurve`` that represents the throughput of the sensor
1039  itself, to be evaluated in post-assembly trimmed detector coordinates.
1040  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1041  A ``TransmissionCurve`` that represents the throughput of the
1042  atmosphere, assumed to be spatially constant.
1043  - ``strayLightData`` : `object`
1044  An opaque object containing calibration information for
1045  stray-light correction. If `None`, no correction will be
1046  performed.
1047  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1048 
1049  Raises
1050  ------
1051  NotImplementedError :
1052  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1053  """
1054  try:
1055  dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1056  dateObs = dateObs.toPython().isoformat()
1057  except RuntimeError:
1058  self.log.warn("Unable to identify dateObs for rawExposure.")
1059  dateObs = None
1060 
1061  ccd = rawExposure.getDetector()
1062  # TODO DM-28093: change this to: rawExposure.getFilterLabel().physicalLabel
1063  filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
1064  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
1065  biasExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.biasDataProductName)
1066  if self.config.doBias else None)
1067  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
1068  linearizer = (dataRef.get("linearizer", immediate=True)
1069  if self.doLinearizedoLinearize(ccd) else None)
1070  if linearizer is not None and not isinstance(linearizer, numpy.ndarray):
1071  linearizer.log = self.log
1072  if isinstance(linearizer, numpy.ndarray):
1073  linearizer = linearize.Linearizer(table=linearizer, detector=ccd)
1074 
1075  crosstalkCalib = None
1076  if self.config.doCrosstalk:
1077  try:
1078  crosstalkCalib = dataRef.get("crosstalk", immediate=True)
1079  except NoResults:
1080  coeffVector = (self.config.crosstalk.crosstalkValues
1081  if self.config.crosstalk.useConfigCoefficients else None)
1082  crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1083  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1084  if self.config.doCrosstalk else None)
1085 
1086  darkExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.darkDataProductName)
1087  if self.config.doDark else None)
1088  flatExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.flatDataProductName,
1089  dateObs=dateObs)
1090  if self.config.doFlat else None)
1091 
1092  brighterFatterKernel = None
1093  brighterFatterGains = None
1094  if self.config.doBrighterFatter is True:
1095  try:
1096  # Use the new-style cp_pipe version of the kernel if it exists
1097  # If using a new-style kernel, always use the self-consistent
1098  # gains, i.e. the ones inside the kernel object itself
1099  brighterFatterKernel = dataRef.get("brighterFatterKernel")
1100  brighterFatterGains = brighterFatterKernel.gain
1101  self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded")
1102  except NoResults:
1103  try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
1104  brighterFatterKernel = dataRef.get("bfKernel")
1105  self.log.info("Old style bright-fatter kernel (np.array) loaded")
1106  except NoResults:
1107  brighterFatterKernel = None
1108  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1109  # If the kernel is not an ndarray, it's the cp_pipe version
1110  # so extract the kernel for this detector, or raise an error
1111  if self.config.brighterFatterLevel == 'DETECTOR':
1112  if brighterFatterKernel.detectorKernel:
1113  brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
1114  elif brighterFatterKernel.detectorKernelFromAmpKernels:
1115  brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
1116  else:
1117  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1118  else:
1119  # TODO DM-15631 for implementing this
1120  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1121 
1122  defectList = (dataRef.get("defects")
1123  if self.config.doDefect else None)
1124  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
1125  if self.config.doAssembleIsrExposures else None)
1126  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1127  else pipeBase.Struct(fringes=None))
1128 
1129  if self.config.doAttachTransmissionCurve:
1130  opticsTransmission = (dataRef.get("transmission_optics")
1131  if self.config.doUseOpticsTransmission else None)
1132  filterTransmission = (dataRef.get("transmission_filter")
1133  if self.config.doUseFilterTransmission else None)
1134  sensorTransmission = (dataRef.get("transmission_sensor")
1135  if self.config.doUseSensorTransmission else None)
1136  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1137  if self.config.doUseAtmosphereTransmission else None)
1138  else:
1139  opticsTransmission = None
1140  filterTransmission = None
1141  sensorTransmission = None
1142  atmosphereTransmission = None
1143 
1144  if self.config.doStrayLight:
1145  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1146  else:
1147  strayLightData = None
1148 
1149  illumMaskedImage = (self.getIsrExposuregetIsrExposure(dataRef,
1150  self.config.illuminationCorrectionDataProductName).getMaskedImage()
1151  if (self.config.doIlluminationCorrection
1152  and filterName in self.config.illumFilters)
1153  else None)
1154 
1155  # Struct should include only kwargs to run()
1156  return pipeBase.Struct(bias=biasExposure,
1157  linearizer=linearizer,
1158  crosstalk=crosstalkCalib,
1159  crosstalkSources=crosstalkSources,
1160  dark=darkExposure,
1161  flat=flatExposure,
1162  bfKernel=brighterFatterKernel,
1163  bfGains=brighterFatterGains,
1164  defects=defectList,
1165  fringes=fringeStruct,
1166  opticsTransmission=opticsTransmission,
1167  filterTransmission=filterTransmission,
1168  sensorTransmission=sensorTransmission,
1169  atmosphereTransmission=atmosphereTransmission,
1170  strayLightData=strayLightData,
1171  illumMaskedImage=illumMaskedImage
1172  )
1173 
1174  @pipeBase.timeMethod
1175  def run(self, ccdExposure, camera=None, bias=None, linearizer=None,
1176  crosstalk=None, crosstalkSources=None,
1177  dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1178  fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1179  sensorTransmission=None, atmosphereTransmission=None,
1180  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1181  isGen3=False,
1182  ):
1183  """Perform instrument signature removal on an exposure.
1184 
1185  Steps included in the ISR processing, in order performed, are:
1186  - saturation and suspect pixel masking
1187  - overscan subtraction
1188  - CCD assembly of individual amplifiers
1189  - bias subtraction
1190  - variance image construction
1191  - linearization of non-linear response
1192  - crosstalk masking
1193  - brighter-fatter correction
1194  - dark subtraction
1195  - fringe correction
1196  - stray light subtraction
1197  - flat correction
1198  - masking of known defects and camera specific features
1199  - vignette calculation
1200  - appending transmission curve and distortion model
1201 
1202  Parameters
1203  ----------
1204  ccdExposure : `lsst.afw.image.Exposure`
1205  The raw exposure that is to be run through ISR. The
1206  exposure is modified by this method.
1207  camera : `lsst.afw.cameraGeom.Camera`, optional
1208  The camera geometry for this exposure. Required if ``isGen3`` is
1209  `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1210  ``flat`` does not have an associated detector.
1211  bias : `lsst.afw.image.Exposure`, optional
1212  Bias calibration frame.
1213  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1214  Functor for linearization.
1215  crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1216  Calibration for crosstalk.
1217  crosstalkSources : `list`, optional
1218  List of possible crosstalk sources.
1219  dark : `lsst.afw.image.Exposure`, optional
1220  Dark calibration frame.
1221  flat : `lsst.afw.image.Exposure`, optional
1222  Flat calibration frame.
1223  bfKernel : `numpy.ndarray`, optional
1224  Brighter-fatter kernel.
1225  bfGains : `dict` of `float`, optional
1226  Gains used to override the detector's nominal gains for the
1227  brighter-fatter correction. A dict keyed by amplifier name for
1228  the detector in question.
1229  defects : `lsst.ip.isr.Defects`, optional
1230  List of defects.
1231  fringes : `lsst.pipe.base.Struct`, optional
1232  Struct containing the fringe correction data, with
1233  elements:
1234  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1235  - ``seed``: random seed derived from the ccdExposureId for random
1236  number generator (`uint32`)
1237  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1238  A ``TransmissionCurve`` that represents the throughput of the optics,
1239  to be evaluated in focal-plane coordinates.
1240  filterTransmission : `lsst.afw.image.TransmissionCurve`
1241  A ``TransmissionCurve`` that represents the throughput of the filter
1242  itself, to be evaluated in focal-plane coordinates.
1243  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1244  A ``TransmissionCurve`` that represents the throughput of the sensor
1245  itself, to be evaluated in post-assembly trimmed detector coordinates.
1246  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1247  A ``TransmissionCurve`` that represents the throughput of the
1248  atmosphere, assumed to be spatially constant.
1249  detectorNum : `int`, optional
1250  The integer number for the detector to process.
1251  isGen3 : bool, optional
1252  Flag this call to run() as using the Gen3 butler environment.
1253  strayLightData : `object`, optional
1254  Opaque object containing calibration information for stray-light
1255  correction. If `None`, no correction will be performed.
1256  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1257  Illumination correction image.
1258 
1259  Returns
1260  -------
1261  result : `lsst.pipe.base.Struct`
1262  Result struct with component:
1263  - ``exposure`` : `afw.image.Exposure`
1264  The fully ISR corrected exposure.
1265  - ``outputExposure`` : `afw.image.Exposure`
1266  An alias for `exposure`
1267  - ``ossThumb`` : `numpy.ndarray`
1268  Thumbnail image of the exposure after overscan subtraction.
1269  - ``flattenedThumb`` : `numpy.ndarray`
1270  Thumbnail image of the exposure after flat-field correction.
1271 
1272  Raises
1273  ------
1274  RuntimeError
1275  Raised if a configuration option is set to True, but the
1276  required calibration data has not been specified.
1277 
1278  Notes
1279  -----
1280  The current processed exposure can be viewed by setting the
1281  appropriate lsstDebug entries in the `debug.display`
1282  dictionary. The names of these entries correspond to some of
1283  the IsrTaskConfig Boolean options, with the value denoting the
1284  frame to use. The exposure is shown inside the matching
1285  option check and after the processing of that step has
1286  finished. The steps with debug points are:
1287 
1288  doAssembleCcd
1289  doBias
1290  doCrosstalk
1291  doBrighterFatter
1292  doDark
1293  doFringe
1294  doStrayLight
1295  doFlat
1296 
1297  In addition, setting the "postISRCCD" entry displays the
1298  exposure after all ISR processing has finished.
1299 
1300  """
1301 
1302  if isGen3 is True:
1303  # Gen3 currently cannot automatically do configuration overrides.
1304  # DM-15257 looks to discuss this issue.
1305  # Configure input exposures;
1306  if detectorNum is None:
1307  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1308 
1309  ccdExposure = self.ensureExposureensureExposure(ccdExposure, camera, detectorNum)
1310  bias = self.ensureExposureensureExposure(bias, camera, detectorNum)
1311  dark = self.ensureExposureensureExposure(dark, camera, detectorNum)
1312  flat = self.ensureExposureensureExposure(flat, camera, detectorNum)
1313  else:
1314  if isinstance(ccdExposure, ButlerDataRef):
1315  return self.runDataRefrunDataRef(ccdExposure)
1316 
1317  ccd = ccdExposure.getDetector()
1318  # TODO DM-28093: change this to: ccdExposure.getFilterLabel().physicalLabel
1319  filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1320 
1321  if not ccd:
1322  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1323  ccd = [FakeAmp(ccdExposure, self.config)]
1324 
1325  # Validate Input
1326  if self.config.doBias and bias is None:
1327  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1328  if self.doLinearizedoLinearize(ccd) and linearizer is None:
1329  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1330  if self.config.doBrighterFatter and bfKernel is None:
1331  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1332  if self.config.doDark and dark is None:
1333  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1334  if self.config.doFlat and flat is None:
1335  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1336  if self.config.doDefect and defects is None:
1337  raise RuntimeError("Must supply defects if config.doDefect=True.")
1338  if (self.config.doFringe and filterName in self.fringe.config.filters
1339  and fringes.fringes is None):
1340  # The `fringes` object needs to be a pipeBase.Struct, as
1341  # we use it as a `dict` for the parameters of
1342  # `FringeTask.run()`. The `fringes.fringes` `list` may
1343  # not be `None` if `doFringe=True`. Otherwise, raise.
1344  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1345  if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters
1346  and illumMaskedImage is None):
1347  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1348 
1349  # Begin ISR processing.
1350  if self.config.doConvertIntToFloat:
1351  self.log.info("Converting exposure to floating point values.")
1352  ccdExposure = self.convertIntToFloatconvertIntToFloat(ccdExposure)
1353 
1354  if self.config.doBias and self.config.doBiasBeforeOverscan:
1355  self.log.info("Applying bias correction.")
1356  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1357  trimToFit=self.config.doTrimToMatchCalib)
1358  self.debugViewdebugView(ccdExposure, "doBias")
1359 
1360  # Amplifier level processing.
1361  overscans = []
1362  for amp in ccd:
1363  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1364  if ccdExposure.getBBox().contains(amp.getBBox()):
1365  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1366  badAmp = self.maskAmplifiermaskAmplifier(ccdExposure, amp, defects)
1367 
1368  if self.config.doOverscan and not badAmp:
1369  # Overscan correction on amp-by-amp basis.
1370  overscanResults = self.overscanCorrectionoverscanCorrection(ccdExposure, amp)
1371  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1372  if overscanResults is not None and \
1373  self.config.qa is not None and self.config.qa.saveStats is True:
1374  if isinstance(overscanResults.overscanFit, float):
1375  qaMedian = overscanResults.overscanFit
1376  qaStdev = float("NaN")
1377  else:
1378  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1379  afwMath.MEDIAN | afwMath.STDEVCLIP)
1380  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1381  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1382 
1383  self.metadata.set(f"FIT MEDIAN {amp.getName()}", qaMedian)
1384  self.metadata.set(f"FIT STDEV {amp.getName()}", qaStdev)
1385  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1386  amp.getName(), qaMedian, qaStdev)
1387 
1388  # Residuals after overscan correction
1389  qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1390  afwMath.MEDIAN | afwMath.STDEVCLIP)
1391  qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1392  qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1393 
1394  self.metadata.set(f"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter)
1395  self.metadata.set(f"RESIDUAL STDEV {amp.getName()}", qaStdevAfter)
1396  self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f",
1397  amp.getName(), qaMedianAfter, qaStdevAfter)
1398 
1399  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1400  else:
1401  if badAmp:
1402  self.log.warn("Amplifier %s is bad.", amp.getName())
1403  overscanResults = None
1404 
1405  overscans.append(overscanResults if overscanResults is not None else None)
1406  else:
1407  self.log.info("Skipped OSCAN for %s.", amp.getName())
1408 
1409  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1410  self.log.info("Applying crosstalk correction.")
1411  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1412  crosstalkSources=crosstalkSources)
1413  self.debugViewdebugView(ccdExposure, "doCrosstalk")
1414 
1415  if self.config.doAssembleCcd:
1416  self.log.info("Assembling CCD from amplifiers.")
1417  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1418 
1419  if self.config.expectWcs and not ccdExposure.getWcs():
1420  self.log.warn("No WCS found in input exposure.")
1421  self.debugViewdebugView(ccdExposure, "doAssembleCcd")
1422 
1423  ossThumb = None
1424  if self.config.qa.doThumbnailOss:
1425  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1426 
1427  if self.config.doBias and not self.config.doBiasBeforeOverscan:
1428  self.log.info("Applying bias correction.")
1429  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1430  trimToFit=self.config.doTrimToMatchCalib)
1431  self.debugViewdebugView(ccdExposure, "doBias")
1432 
1433  if self.config.doVariance:
1434  for amp, overscanResults in zip(ccd, overscans):
1435  if ccdExposure.getBBox().contains(amp.getBBox()):
1436  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1437  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1438  if overscanResults is not None:
1439  self.updateVarianceupdateVariance(ampExposure, amp,
1440  overscanImage=overscanResults.overscanImage)
1441  else:
1442  self.updateVarianceupdateVariance(ampExposure, amp,
1443  overscanImage=None)
1444  if self.config.qa is not None and self.config.qa.saveStats is True:
1445  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1446  afwMath.MEDIAN | afwMath.STDEVCLIP)
1447  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1448  qaStats.getValue(afwMath.MEDIAN))
1449  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1450  qaStats.getValue(afwMath.STDEVCLIP))
1451  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1452  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1453  qaStats.getValue(afwMath.STDEVCLIP))
1454 
1455  if self.doLinearizedoLinearize(ccd):
1456  self.log.info("Applying linearizer.")
1457  linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1458  detector=ccd, log=self.log)
1459 
1460  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1461  self.log.info("Applying crosstalk correction.")
1462  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1463  crosstalkSources=crosstalkSources, isTrimmed=True)
1464  self.debugViewdebugView(ccdExposure, "doCrosstalk")
1465 
1466  # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1467  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1468  if self.config.doDefect:
1469  self.log.info("Masking defects.")
1470  self.maskDefectmaskDefect(ccdExposure, defects)
1471 
1472  if self.config.numEdgeSuspect > 0:
1473  self.log.info("Masking edges as SUSPECT.")
1474  self.maskEdgesmaskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1475  maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
1476 
1477  if self.config.doNanMasking:
1478  self.log.info("Masking NAN value pixels.")
1479  self.maskNanmaskNan(ccdExposure)
1480 
1481  if self.config.doWidenSaturationTrails:
1482  self.log.info("Widening saturation trails.")
1483  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1484 
1485  if self.config.doCameraSpecificMasking:
1486  self.log.info("Masking regions for camera specific reasons.")
1487  self.masking.run(ccdExposure)
1488 
1489  if self.config.doBrighterFatter:
1490  # We need to apply flats and darks before we can interpolate, and we
1491  # need to interpolate before we do B-F, but we do B-F without the
1492  # flats and darks applied so we can work in units of electrons or holes.
1493  # This context manager applies and then removes the darks and flats.
1494  #
1495  # We also do not want to interpolate values here, so operate on temporary
1496  # images so we can apply only the BF-correction and roll back the
1497  # interpolation.
1498  interpExp = ccdExposure.clone()
1499  with self.flatContextflatContext(interpExp, flat, dark):
1500  isrFunctions.interpolateFromMask(
1501  maskedImage=interpExp.getMaskedImage(),
1502  fwhm=self.config.fwhm,
1503  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1504  maskNameList=self.config.maskListToInterpolate
1505  )
1506  bfExp = interpExp.clone()
1507 
1508  self.log.info("Applying brighter fatter correction using kernel type %s / gains %s.",
1509  type(bfKernel), type(bfGains))
1510  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1511  self.config.brighterFatterMaxIter,
1512  self.config.brighterFatterThreshold,
1513  self.config.brighterFatterApplyGain,
1514  bfGains)
1515  if bfResults[1] == self.config.brighterFatterMaxIter:
1516  self.log.warn("Brighter fatter correction did not converge, final difference %f.",
1517  bfResults[0])
1518  else:
1519  self.log.info("Finished brighter fatter correction in %d iterations.",
1520  bfResults[1])
1521  image = ccdExposure.getMaskedImage().getImage()
1522  bfCorr = bfExp.getMaskedImage().getImage()
1523  bfCorr -= interpExp.getMaskedImage().getImage()
1524  image += bfCorr
1525 
1526  # Applying the brighter-fatter correction applies a
1527  # convolution to the science image. At the edges this
1528  # convolution may not have sufficient valid pixels to
1529  # produce a valid correction. Mark pixels within the size
1530  # of the brighter-fatter kernel as EDGE to warn of this
1531  # fact.
1532  self.log.info("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1533  self.maskEdgesmaskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1534  maskPlane="EDGE")
1535 
1536  if self.config.brighterFatterMaskGrowSize > 0:
1537  self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1538  for maskPlane in self.config.maskListToInterpolate:
1539  isrFunctions.growMasks(ccdExposure.getMask(),
1540  radius=self.config.brighterFatterMaskGrowSize,
1541  maskNameList=maskPlane,
1542  maskValue=maskPlane)
1543 
1544  self.debugViewdebugView(ccdExposure, "doBrighterFatter")
1545 
1546  if self.config.doDark:
1547  self.log.info("Applying dark correction.")
1548  self.darkCorrectiondarkCorrection(ccdExposure, dark)
1549  self.debugViewdebugView(ccdExposure, "doDark")
1550 
1551  if self.config.doFringe and not self.config.fringeAfterFlat:
1552  self.log.info("Applying fringe correction before flat.")
1553  self.fringe.run(ccdExposure, **fringes.getDict())
1554  self.debugViewdebugView(ccdExposure, "doFringe")
1555 
1556  if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1557  self.log.info("Checking strayLight correction.")
1558  self.strayLight.run(ccdExposure, strayLightData)
1559  self.debugViewdebugView(ccdExposure, "doStrayLight")
1560 
1561  if self.config.doFlat:
1562  self.log.info("Applying flat correction.")
1563  self.flatCorrectionflatCorrection(ccdExposure, flat)
1564  self.debugViewdebugView(ccdExposure, "doFlat")
1565 
1566  if self.config.doApplyGains:
1567  self.log.info("Applying gain correction instead of flat.")
1568  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1569 
1570  if self.config.doFringe and self.config.fringeAfterFlat:
1571  self.log.info("Applying fringe correction after flat.")
1572  self.fringe.run(ccdExposure, **fringes.getDict())
1573 
1574  if self.config.doVignette:
1575  self.log.info("Constructing Vignette polygon.")
1576  self.vignettePolygonvignettePolygon = self.vignette.run(ccdExposure)
1577 
1578  if self.config.vignette.doWriteVignettePolygon:
1579  self.setValidPolygonIntersectsetValidPolygonIntersect(ccdExposure, self.vignettePolygonvignettePolygon)
1580 
1581  if self.config.doAttachTransmissionCurve:
1582  self.log.info("Adding transmission curves.")
1583  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1584  filterTransmission=filterTransmission,
1585  sensorTransmission=sensorTransmission,
1586  atmosphereTransmission=atmosphereTransmission)
1587 
1588  flattenedThumb = None
1589  if self.config.qa.doThumbnailFlattened:
1590  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1591 
1592  if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1593  self.log.info("Performing illumination correction.")
1594  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1595  illumMaskedImage, illumScale=self.config.illumScale,
1596  trimToFit=self.config.doTrimToMatchCalib)
1597 
1598  preInterpExp = None
1599  if self.config.doSaveInterpPixels:
1600  preInterpExp = ccdExposure.clone()
1601 
1602  # Reset and interpolate bad pixels.
1603  #
1604  # Large contiguous bad regions (which should have the BAD mask
1605  # bit set) should have their values set to the image median.
1606  # This group should include defects and bad amplifiers. As the
1607  # area covered by these defects are large, there's little
1608  # reason to expect that interpolation would provide a more
1609  # useful value.
1610  #
1611  # Smaller defects can be safely interpolated after the larger
1612  # regions have had their pixel values reset. This ensures
1613  # that the remaining defects adjacent to bad amplifiers (as an
1614  # example) do not attempt to interpolate extreme values.
1615  if self.config.doSetBadRegions:
1616  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1617  if badPixelCount > 0:
1618  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1619 
1620  if self.config.doInterpolate:
1621  self.log.info("Interpolating masked pixels.")
1622  isrFunctions.interpolateFromMask(
1623  maskedImage=ccdExposure.getMaskedImage(),
1624  fwhm=self.config.fwhm,
1625  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1626  maskNameList=list(self.config.maskListToInterpolate)
1627  )
1628 
1629  self.roughZeroPointroughZeroPoint(ccdExposure)
1630 
1631  if self.config.doMeasureBackground:
1632  self.log.info("Measuring background level.")
1633  self.measureBackgroundmeasureBackground(ccdExposure, self.config.qa)
1634 
1635  if self.config.qa is not None and self.config.qa.saveStats is True:
1636  for amp in ccd:
1637  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1638  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1639  afwMath.MEDIAN | afwMath.STDEVCLIP)
1640  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1641  qaStats.getValue(afwMath.MEDIAN))
1642  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1643  qaStats.getValue(afwMath.STDEVCLIP))
1644  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1645  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1646  qaStats.getValue(afwMath.STDEVCLIP))
1647 
1648  self.debugViewdebugView(ccdExposure, "postISRCCD")
1649 
1650  return pipeBase.Struct(
1651  exposure=ccdExposure,
1652  ossThumb=ossThumb,
1653  flattenedThumb=flattenedThumb,
1654 
1655  preInterpolatedExposure=preInterpExp,
1656  outputExposure=ccdExposure,
1657  outputOssThumbnail=ossThumb,
1658  outputFlattenedThumbnail=flattenedThumb,
1659  )
1660 
1661  @pipeBase.timeMethod
1662  def runDataRef(self, sensorRef):
1663  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1664 
1665  This method contains the `CmdLineTask` interface to the ISR
1666  processing. All IO is handled here, freeing the `run()` method
1667  to manage only pixel-level calculations. The steps performed
1668  are:
1669  - Read in necessary detrending/isr/calibration data.
1670  - Process raw exposure in `run()`.
1671  - Persist the ISR-corrected exposure as "postISRCCD" if
1672  config.doWrite=True.
1673 
1674  Parameters
1675  ----------
1676  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1677  DataRef of the detector data to be processed
1678 
1679  Returns
1680  -------
1681  result : `lsst.pipe.base.Struct`
1682  Result struct with component:
1683  - ``exposure`` : `afw.image.Exposure`
1684  The fully ISR corrected exposure.
1685 
1686  Raises
1687  ------
1688  RuntimeError
1689  Raised if a configuration option is set to True, but the
1690  required calibration data does not exist.
1691 
1692  """
1693  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1694 
1695  ccdExposure = sensorRef.get(self.config.datasetType)
1696 
1697  camera = sensorRef.get("camera")
1698  isrData = self.readIsrDatareadIsrData(sensorRef, ccdExposure)
1699 
1700  result = self.runrun(ccdExposure, camera=camera, **isrData.getDict())
1701 
1702  if self.config.doWrite:
1703  sensorRef.put(result.exposure, "postISRCCD")
1704  if result.preInterpolatedExposure is not None:
1705  sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1706  if result.ossThumb is not None:
1707  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1708  if result.flattenedThumb is not None:
1709  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1710 
1711  return result
1712 
1713  def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1714  """Retrieve a calibration dataset for removing instrument signature.
1715 
1716  Parameters
1717  ----------
1718 
1719  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1720  DataRef of the detector data to find calibration datasets
1721  for.
1722  datasetType : `str`
1723  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1724  dateObs : `str`, optional
1725  Date of the observation. Used to correct butler failures
1726  when using fallback filters.
1727  immediate : `Bool`
1728  If True, disable butler proxies to enable error handling
1729  within this routine.
1730 
1731  Returns
1732  -------
1733  exposure : `lsst.afw.image.Exposure`
1734  Requested calibration frame.
1735 
1736  Raises
1737  ------
1738  RuntimeError
1739  Raised if no matching calibration frame can be found.
1740  """
1741  try:
1742  exp = dataRef.get(datasetType, immediate=immediate)
1743  except Exception as exc1:
1744  if not self.config.fallbackFilterName:
1745  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1746  try:
1747  if self.config.useFallbackDate and dateObs:
1748  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1749  dateObs=dateObs, immediate=immediate)
1750  else:
1751  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1752  except Exception as exc2:
1753  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1754  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1755  self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1756 
1757  if self.config.doAssembleIsrExposures:
1758  exp = self.assembleCcd.assembleCcd(exp)
1759  return exp
1760 
1761  def ensureExposure(self, inputExp, camera, detectorNum):
1762  """Ensure that the data returned by Butler is a fully constructed exposure.
1763 
1764  ISR requires exposure-level image data for historical reasons, so if we did
1765  not recieve that from Butler, construct it from what we have, modifying the
1766  input in place.
1767 
1768  Parameters
1769  ----------
1770  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1771  `lsst.afw.image.ImageF`
1772  The input data structure obtained from Butler.
1773  camera : `lsst.afw.cameraGeom.camera`
1774  The camera associated with the image. Used to find the appropriate
1775  detector.
1776  detectorNum : `int`
1777  The detector this exposure should match.
1778 
1779  Returns
1780  -------
1781  inputExp : `lsst.afw.image.Exposure`
1782  The re-constructed exposure, with appropriate detector parameters.
1783 
1784  Raises
1785  ------
1786  TypeError
1787  Raised if the input data cannot be used to construct an exposure.
1788  """
1789  if isinstance(inputExp, afwImage.DecoratedImageU):
1790  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1791  elif isinstance(inputExp, afwImage.ImageF):
1792  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1793  elif isinstance(inputExp, afwImage.MaskedImageF):
1794  inputExp = afwImage.makeExposure(inputExp)
1795  elif isinstance(inputExp, afwImage.Exposure):
1796  pass
1797  elif inputExp is None:
1798  # Assume this will be caught by the setup if it is a problem.
1799  return inputExp
1800  else:
1801  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1802  (type(inputExp), ))
1803 
1804  if inputExp.getDetector() is None:
1805  inputExp.setDetector(camera[detectorNum])
1806 
1807  return inputExp
1808 
1809  def convertIntToFloat(self, exposure):
1810  """Convert exposure image from uint16 to float.
1811 
1812  If the exposure does not need to be converted, the input is
1813  immediately returned. For exposures that are converted to use
1814  floating point pixels, the variance is set to unity and the
1815  mask to zero.
1816 
1817  Parameters
1818  ----------
1819  exposure : `lsst.afw.image.Exposure`
1820  The raw exposure to be converted.
1821 
1822  Returns
1823  -------
1824  newexposure : `lsst.afw.image.Exposure`
1825  The input ``exposure``, converted to floating point pixels.
1826 
1827  Raises
1828  ------
1829  RuntimeError
1830  Raised if the exposure type cannot be converted to float.
1831 
1832  """
1833  if isinstance(exposure, afwImage.ExposureF):
1834  # Nothing to be done
1835  self.log.debug("Exposure already of type float.")
1836  return exposure
1837  if not hasattr(exposure, "convertF"):
1838  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1839 
1840  newexposure = exposure.convertF()
1841  newexposure.variance[:] = 1
1842  newexposure.mask[:] = 0x0
1843 
1844  return newexposure
1845 
1846  def maskAmplifier(self, ccdExposure, amp, defects):
1847  """Identify bad amplifiers, saturated and suspect pixels.
1848 
1849  Parameters
1850  ----------
1851  ccdExposure : `lsst.afw.image.Exposure`
1852  Input exposure to be masked.
1853  amp : `lsst.afw.table.AmpInfoCatalog`
1854  Catalog of parameters defining the amplifier on this
1855  exposure to mask.
1856  defects : `lsst.ip.isr.Defects`
1857  List of defects. Used to determine if the entire
1858  amplifier is bad.
1859 
1860  Returns
1861  -------
1862  badAmp : `Bool`
1863  If this is true, the entire amplifier area is covered by
1864  defects and unusable.
1865 
1866  """
1867  maskedImage = ccdExposure.getMaskedImage()
1868 
1869  badAmp = False
1870 
1871  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1872  # comparison with current defects definition.
1873  if defects is not None:
1874  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1875 
1876  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1877  # association with pixels in current ccdExposure).
1878  if badAmp:
1879  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1880  afwImage.PARENT)
1881  maskView = dataView.getMask()
1882  maskView |= maskView.getPlaneBitMask("BAD")
1883  del maskView
1884  return badAmp
1885 
1886  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1887  # Saturation and suspect pixels can be masked now, though.
1888  limits = dict()
1889  if self.config.doSaturation and not badAmp:
1890  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1891  if self.config.doSuspect and not badAmp:
1892  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1893  if math.isfinite(self.config.saturation):
1894  limits.update({self.config.saturatedMaskName: self.config.saturation})
1895 
1896  for maskName, maskThreshold in limits.items():
1897  if not math.isnan(maskThreshold):
1898  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1899  isrFunctions.makeThresholdMask(
1900  maskedImage=dataView,
1901  threshold=maskThreshold,
1902  growFootprints=0,
1903  maskName=maskName
1904  )
1905 
1906  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1907  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1908  afwImage.PARENT)
1909  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1910  self.config.suspectMaskName])
1911  if numpy.all(maskView.getArray() & maskVal > 0):
1912  badAmp = True
1913  maskView |= maskView.getPlaneBitMask("BAD")
1914 
1915  return badAmp
1916 
1917  def overscanCorrection(self, ccdExposure, amp):
1918  """Apply overscan correction in place.
1919 
1920  This method does initial pixel rejection of the overscan
1921  region. The overscan can also be optionally segmented to
1922  allow for discontinuous overscan responses to be fit
1923  separately. The actual overscan subtraction is performed by
1924  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1925  which is called here after the amplifier is preprocessed.
1926 
1927  Parameters
1928  ----------
1929  ccdExposure : `lsst.afw.image.Exposure`
1930  Exposure to have overscan correction performed.
1931  amp : `lsst.afw.cameraGeom.Amplifer`
1932  The amplifier to consider while correcting the overscan.
1933 
1934  Returns
1935  -------
1936  overscanResults : `lsst.pipe.base.Struct`
1937  Result struct with components:
1938  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1939  Value or fit subtracted from the amplifier image data.
1940  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1941  Value or fit subtracted from the overscan image data.
1942  - ``overscanImage`` : `lsst.afw.image.Image`
1943  Image of the overscan region with the overscan
1944  correction applied. This quantity is used to estimate
1945  the amplifier read noise empirically.
1946 
1947  Raises
1948  ------
1949  RuntimeError
1950  Raised if the ``amp`` does not contain raw pixel information.
1951 
1952  See Also
1953  --------
1954  lsst.ip.isr.isrFunctions.overscanCorrection
1955  """
1956  if amp.getRawHorizontalOverscanBBox().isEmpty():
1957  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1958  return None
1959 
1960  statControl = afwMath.StatisticsControl()
1961  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1962 
1963  # Determine the bounding boxes
1964  dataBBox = amp.getRawDataBBox()
1965  oscanBBox = amp.getRawHorizontalOverscanBBox()
1966  dx0 = 0
1967  dx1 = 0
1968 
1969  prescanBBox = amp.getRawPrescanBBox()
1970  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1971  dx0 += self.config.overscanNumLeadingColumnsToSkip
1972  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1973  else:
1974  dx0 += self.config.overscanNumTrailingColumnsToSkip
1975  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1976 
1977  # Determine if we need to work on subregions of the amplifier and overscan.
1978  imageBBoxes = []
1979  overscanBBoxes = []
1980 
1981  if ((self.config.overscanBiasJump
1982  and self.config.overscanBiasJumpLocation)
1983  and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
1984  and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1985  self.config.overscanBiasJumpDevices)):
1986  if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
1987  yLower = self.config.overscanBiasJumpLocation
1988  yUpper = dataBBox.getHeight() - yLower
1989  else:
1990  yUpper = self.config.overscanBiasJumpLocation
1991  yLower = dataBBox.getHeight() - yUpper
1992 
1993  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1994  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1995  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1996  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1997  yLower)))
1998 
1999  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
2000  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
2001  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
2002  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2003  yUpper)))
2004  else:
2005  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2006  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
2007  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2008  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2009  oscanBBox.getHeight())))
2010 
2011  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
2012  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
2013  ampImage = ccdExposure.maskedImage[imageBBox]
2014  overscanImage = ccdExposure.maskedImage[overscanBBox]
2015 
2016  overscanArray = overscanImage.image.array
2017  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2018  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2019  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
2020 
2021  statControl = afwMath.StatisticsControl()
2022  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2023 
2024  overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp)
2025 
2026  # Measure average overscan levels and record them in the metadata.
2027  levelStat = afwMath.MEDIAN
2028  sigmaStat = afwMath.STDEVCLIP
2029 
2030  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2031  self.config.qa.flatness.nIter)
2032  metadata = ccdExposure.getMetadata()
2033  ampNum = amp.getName()
2034  # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
2035  if isinstance(overscanResults.overscanFit, float):
2036  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2037  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2038  else:
2039  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2040  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2041  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2042 
2043  return overscanResults
2044 
2045  def updateVariance(self, ampExposure, amp, overscanImage=None):
2046  """Set the variance plane using the amplifier gain and read noise
2047 
2048  The read noise is calculated from the ``overscanImage`` if the
2049  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2050  the value from the amplifier data is used.
2051 
2052  Parameters
2053  ----------
2054  ampExposure : `lsst.afw.image.Exposure`
2055  Exposure to process.
2056  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2057  Amplifier detector data.
2058  overscanImage : `lsst.afw.image.MaskedImage`, optional.
2059  Image of overscan, required only for empirical read noise.
2060 
2061  See also
2062  --------
2063  lsst.ip.isr.isrFunctions.updateVariance
2064  """
2065  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2066  gain = amp.getGain()
2067 
2068  if math.isnan(gain):
2069  gain = 1.0
2070  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2071  elif gain <= 0:
2072  patchedGain = 1.0
2073  self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
2074  amp.getName(), gain, patchedGain)
2075  gain = patchedGain
2076 
2077  if self.config.doEmpiricalReadNoise and overscanImage is None:
2078  self.log.info("Overscan is none for EmpiricalReadNoise.")
2079 
2080  if self.config.doEmpiricalReadNoise and overscanImage is not None:
2081  stats = afwMath.StatisticsControl()
2082  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2083  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2084  self.log.info("Calculated empirical read noise for amp %s: %f.",
2085  amp.getName(), readNoise)
2086  else:
2087  readNoise = amp.getReadNoise()
2088 
2089  isrFunctions.updateVariance(
2090  maskedImage=ampExposure.getMaskedImage(),
2091  gain=gain,
2092  readNoise=readNoise,
2093  )
2094 
2095  def darkCorrection(self, exposure, darkExposure, invert=False):
2096  """Apply dark correction in place.
2097 
2098  Parameters
2099  ----------
2100  exposure : `lsst.afw.image.Exposure`
2101  Exposure to process.
2102  darkExposure : `lsst.afw.image.Exposure`
2103  Dark exposure of the same size as ``exposure``.
2104  invert : `Bool`, optional
2105  If True, re-add the dark to an already corrected image.
2106 
2107  Raises
2108  ------
2109  RuntimeError
2110  Raised if either ``exposure`` or ``darkExposure`` do not
2111  have their dark time defined.
2112 
2113  See Also
2114  --------
2115  lsst.ip.isr.isrFunctions.darkCorrection
2116  """
2117  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2118  if math.isnan(expScale):
2119  raise RuntimeError("Exposure darktime is NAN.")
2120  if darkExposure.getInfo().getVisitInfo() is not None \
2121  and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2122  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2123  else:
2124  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
2125  # so getDarkTime() does not exist.
2126  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2127  darkScale = 1.0
2128 
2129  isrFunctions.darkCorrection(
2130  maskedImage=exposure.getMaskedImage(),
2131  darkMaskedImage=darkExposure.getMaskedImage(),
2132  expScale=expScale,
2133  darkScale=darkScale,
2134  invert=invert,
2135  trimToFit=self.config.doTrimToMatchCalib
2136  )
2137 
2138  def doLinearize(self, detector):
2139  """Check if linearization is needed for the detector cameraGeom.
2140 
2141  Checks config.doLinearize and the linearity type of the first
2142  amplifier.
2143 
2144  Parameters
2145  ----------
2146  detector : `lsst.afw.cameraGeom.Detector`
2147  Detector to get linearity type from.
2148 
2149  Returns
2150  -------
2151  doLinearize : `Bool`
2152  If True, linearization should be performed.
2153  """
2154  return self.config.doLinearize and \
2155  detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2156 
2157  def flatCorrection(self, exposure, flatExposure, invert=False):
2158  """Apply flat correction in place.
2159 
2160  Parameters
2161  ----------
2162  exposure : `lsst.afw.image.Exposure`
2163  Exposure to process.
2164  flatExposure : `lsst.afw.image.Exposure`
2165  Flat exposure of the same size as ``exposure``.
2166  invert : `Bool`, optional
2167  If True, unflatten an already flattened image.
2168 
2169  See Also
2170  --------
2171  lsst.ip.isr.isrFunctions.flatCorrection
2172  """
2173  isrFunctions.flatCorrection(
2174  maskedImage=exposure.getMaskedImage(),
2175  flatMaskedImage=flatExposure.getMaskedImage(),
2176  scalingType=self.config.flatScalingType,
2177  userScale=self.config.flatUserScale,
2178  invert=invert,
2179  trimToFit=self.config.doTrimToMatchCalib
2180  )
2181 
2182  def saturationDetection(self, exposure, amp):
2183  """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2184 
2185  Parameters
2186  ----------
2187  exposure : `lsst.afw.image.Exposure`
2188  Exposure to process. Only the amplifier DataSec is processed.
2189  amp : `lsst.afw.table.AmpInfoCatalog`
2190  Amplifier detector data.
2191 
2192  See Also
2193  --------
2194  lsst.ip.isr.isrFunctions.makeThresholdMask
2195  """
2196  if not math.isnan(amp.getSaturation()):
2197  maskedImage = exposure.getMaskedImage()
2198  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2199  isrFunctions.makeThresholdMask(
2200  maskedImage=dataView,
2201  threshold=amp.getSaturation(),
2202  growFootprints=0,
2203  maskName=self.config.saturatedMaskName,
2204  )
2205 
2206  def saturationInterpolation(self, exposure):
2207  """Interpolate over saturated pixels, in place.
2208 
2209  This method should be called after `saturationDetection`, to
2210  ensure that the saturated pixels have been identified in the
2211  SAT mask. It should also be called after `assembleCcd`, since
2212  saturated regions may cross amplifier boundaries.
2213 
2214  Parameters
2215  ----------
2216  exposure : `lsst.afw.image.Exposure`
2217  Exposure to process.
2218 
2219  See Also
2220  --------
2221  lsst.ip.isr.isrTask.saturationDetection
2222  lsst.ip.isr.isrFunctions.interpolateFromMask
2223  """
2224  isrFunctions.interpolateFromMask(
2225  maskedImage=exposure.getMaskedImage(),
2226  fwhm=self.config.fwhm,
2227  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2228  maskNameList=list(self.config.saturatedMaskName),
2229  )
2230 
2231  def suspectDetection(self, exposure, amp):
2232  """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2233 
2234  Parameters
2235  ----------
2236  exposure : `lsst.afw.image.Exposure`
2237  Exposure to process. Only the amplifier DataSec is processed.
2238  amp : `lsst.afw.table.AmpInfoCatalog`
2239  Amplifier detector data.
2240 
2241  See Also
2242  --------
2243  lsst.ip.isr.isrFunctions.makeThresholdMask
2244 
2245  Notes
2246  -----
2247  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2248  This is intended to indicate pixels that may be affected by unknown systematics;
2249  for example if non-linearity corrections above a certain level are unstable
2250  then that would be a useful value for suspectLevel. A value of `nan` indicates
2251  that no such level exists and no pixels are to be masked as suspicious.
2252  """
2253  suspectLevel = amp.getSuspectLevel()
2254  if math.isnan(suspectLevel):
2255  return
2256 
2257  maskedImage = exposure.getMaskedImage()
2258  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2259  isrFunctions.makeThresholdMask(
2260  maskedImage=dataView,
2261  threshold=suspectLevel,
2262  growFootprints=0,
2263  maskName=self.config.suspectMaskName,
2264  )
2265 
2266  def maskDefect(self, exposure, defectBaseList):
2267  """Mask defects using mask plane "BAD", in place.
2268 
2269  Parameters
2270  ----------
2271  exposure : `lsst.afw.image.Exposure`
2272  Exposure to process.
2273  defectBaseList : `lsst.ip.isr.Defects` or `list` of
2274  `lsst.afw.image.DefectBase`.
2275  List of defects to mask.
2276 
2277  Notes
2278  -----
2279  Call this after CCD assembly, since defects may cross amplifier boundaries.
2280  """
2281  maskedImage = exposure.getMaskedImage()
2282  if not isinstance(defectBaseList, Defects):
2283  # Promotes DefectBase to Defect
2284  defectList = Defects(defectBaseList)
2285  else:
2286  defectList = defectBaseList
2287  defectList.maskPixels(maskedImage, maskName="BAD")
2288 
2289  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2290  """Mask edge pixels with applicable mask plane.
2291 
2292  Parameters
2293  ----------
2294  exposure : `lsst.afw.image.Exposure`
2295  Exposure to process.
2296  numEdgePixels : `int`, optional
2297  Number of edge pixels to mask.
2298  maskPlane : `str`, optional
2299  Mask plane name to use.
2300  level : `str`, optional
2301  Level at which to mask edges.
2302  """
2303  maskedImage = exposure.getMaskedImage()
2304  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2305 
2306  if numEdgePixels > 0:
2307  if level == 'DETECTOR':
2308  boxes = [maskedImage.getBBox()]
2309  elif level == 'AMP':
2310  boxes = [amp.getBBox() for amp in exposure.getDetector()]
2311 
2312  for box in boxes:
2313  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2314  subImage = maskedImage[box]
2315  box.grow(-numEdgePixels)
2316  # Mask pixels outside box
2317  SourceDetectionTask.setEdgeBits(
2318  subImage,
2319  box,
2320  maskBitMask)
2321 
2322  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2323  """Mask and interpolate defects using mask plane "BAD", in place.
2324 
2325  Parameters
2326  ----------
2327  exposure : `lsst.afw.image.Exposure`
2328  Exposure to process.
2329  defectBaseList : `lsst.ip.isr.Defects` or `list` of
2330  `lsst.afw.image.DefectBase`.
2331  List of defects to mask and interpolate.
2332 
2333  See Also
2334  --------
2335  lsst.ip.isr.isrTask.maskDefect
2336  """
2337  self.maskDefectmaskDefect(exposure, defectBaseList)
2338  self.maskEdgesmaskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2339  maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2340  isrFunctions.interpolateFromMask(
2341  maskedImage=exposure.getMaskedImage(),
2342  fwhm=self.config.fwhm,
2343  growSaturatedFootprints=0,
2344  maskNameList=["BAD"],
2345  )
2346 
2347  def maskNan(self, exposure):
2348  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2349 
2350  Parameters
2351  ----------
2352  exposure : `lsst.afw.image.Exposure`
2353  Exposure to process.
2354 
2355  Notes
2356  -----
2357  We mask over all NaNs, including those that are masked with
2358  other bits (because those may or may not be interpolated over
2359  later, and we want to remove all NaNs). Despite this
2360  behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2361  the historical name.
2362  """
2363  maskedImage = exposure.getMaskedImage()
2364 
2365  # Find and mask NaNs
2366  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2367  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2368  numNans = maskNans(maskedImage, maskVal)
2369  self.metadata.set("NUMNANS", numNans)
2370  if numNans > 0:
2371  self.log.warn("There were %d unmasked NaNs.", numNans)
2372 
2373  def maskAndInterpolateNan(self, exposure):
2374  """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2375 
2376  Parameters
2377  ----------
2378  exposure : `lsst.afw.image.Exposure`
2379  Exposure to process.
2380 
2381  See Also
2382  --------
2383  lsst.ip.isr.isrTask.maskNan
2384  """
2385  self.maskNanmaskNan(exposure)
2386  isrFunctions.interpolateFromMask(
2387  maskedImage=exposure.getMaskedImage(),
2388  fwhm=self.config.fwhm,
2389  growSaturatedFootprints=0,
2390  maskNameList=["UNMASKEDNAN"],
2391  )
2392 
2393  def measureBackground(self, exposure, IsrQaConfig=None):
2394  """Measure the image background in subgrids, for quality control purposes.
2395 
2396  Parameters
2397  ----------
2398  exposure : `lsst.afw.image.Exposure`
2399  Exposure to process.
2400  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2401  Configuration object containing parameters on which background
2402  statistics and subgrids to use.
2403  """
2404  if IsrQaConfig is not None:
2405  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2406  IsrQaConfig.flatness.nIter)
2407  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2408  statsControl.setAndMask(maskVal)
2409  maskedImage = exposure.getMaskedImage()
2410  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2411  skyLevel = stats.getValue(afwMath.MEDIAN)
2412  skySigma = stats.getValue(afwMath.STDEVCLIP)
2413  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2414  metadata = exposure.getMetadata()
2415  metadata.set('SKYLEVEL', skyLevel)
2416  metadata.set('SKYSIGMA', skySigma)
2417 
2418  # calcluating flatlevel over the subgrids
2419  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2420  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2421  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2422  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2423  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2424  skyLevels = numpy.zeros((nX, nY))
2425 
2426  for j in range(nY):
2427  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2428  for i in range(nX):
2429  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2430 
2431  xLLC = xc - meshXHalf
2432  yLLC = yc - meshYHalf
2433  xURC = xc + meshXHalf - 1
2434  yURC = yc + meshYHalf - 1
2435 
2436  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2437  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2438 
2439  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2440 
2441  good = numpy.where(numpy.isfinite(skyLevels))
2442  skyMedian = numpy.median(skyLevels[good])
2443  flatness = (skyLevels[good] - skyMedian) / skyMedian
2444  flatness_rms = numpy.std(flatness)
2445  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2446 
2447  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2448  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2449  nX, nY, flatness_pp, flatness_rms)
2450 
2451  metadata.set('FLATNESS_PP', float(flatness_pp))
2452  metadata.set('FLATNESS_RMS', float(flatness_rms))
2453  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2454  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2455  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2456 
2457  def roughZeroPoint(self, exposure):
2458  """Set an approximate magnitude zero point for the exposure.
2459 
2460  Parameters
2461  ----------
2462  exposure : `lsst.afw.image.Exposure`
2463  Exposure to process.
2464  """
2465  # TODO DM-28093: change this to: exposure.getFilterLabel().physicalLabel
2466  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2467  if filterName in self.config.fluxMag0T1:
2468  fluxMag0 = self.config.fluxMag0T1[filterName]
2469  else:
2470  self.log.warn("No rough magnitude zero point set for filter %s.", filterName)
2471  fluxMag0 = self.config.defaultFluxMag0T1
2472 
2473  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2474  if not expTime > 0: # handle NaN as well as <= 0
2475  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2476  return
2477 
2478  self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2479  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2480 
2481  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2482  """Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2483 
2484  Parameters
2485  ----------
2486  ccdExposure : `lsst.afw.image.Exposure`
2487  Exposure to process.
2488  fpPolygon : `lsst.afw.geom.Polygon`
2489  Polygon in focal plane coordinates.
2490  """
2491  # Get ccd corners in focal plane coordinates
2492  ccd = ccdExposure.getDetector()
2493  fpCorners = ccd.getCorners(FOCAL_PLANE)
2494  ccdPolygon = Polygon(fpCorners)
2495 
2496  # Get intersection of ccd corners with fpPolygon
2497  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2498 
2499  # Transform back to pixel positions and build new polygon
2500  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2501  validPolygon = Polygon(ccdPoints)
2502  ccdExposure.getInfo().setValidPolygon(validPolygon)
2503 
2504  @contextmanager
2505  def flatContext(self, exp, flat, dark=None):
2506  """Context manager that applies and removes flats and darks,
2507  if the task is configured to apply them.
2508 
2509  Parameters
2510  ----------
2511  exp : `lsst.afw.image.Exposure`
2512  Exposure to process.
2513  flat : `lsst.afw.image.Exposure`
2514  Flat exposure the same size as ``exp``.
2515  dark : `lsst.afw.image.Exposure`, optional
2516  Dark exposure the same size as ``exp``.
2517 
2518  Yields
2519  ------
2520  exp : `lsst.afw.image.Exposure`
2521  The flat and dark corrected exposure.
2522  """
2523  if self.config.doDark and dark is not None:
2524  self.darkCorrectiondarkCorrection(exp, dark)
2525  if self.config.doFlat:
2526  self.flatCorrectionflatCorrection(exp, flat)
2527  try:
2528  yield exp
2529  finally:
2530  if self.config.doFlat:
2531  self.flatCorrectionflatCorrection(exp, flat, invert=True)
2532  if self.config.doDark and dark is not None:
2533  self.darkCorrectiondarkCorrection(exp, dark, invert=True)
2534 
2535  def debugView(self, exposure, stepname):
2536  """Utility function to examine ISR exposure at different stages.
2537 
2538  Parameters
2539  ----------
2540  exposure : `lsst.afw.image.Exposure`
2541  Exposure to view.
2542  stepname : `str`
2543  State of processing to view.
2544  """
2545  frame = getDebugFrame(self._display, stepname)
2546  if frame:
2547  display = getDisplay(frame)
2548  display.scale('asinh', 'zscale')
2549  display.mtv(exposure)
2550  prompt = "Press Enter to continue [c]... "
2551  while True:
2552  ans = input(prompt).lower()
2553  if ans in ("", "c",):
2554  break
2555 
2556 
2557 class FakeAmp(object):
2558  """A Detector-like object that supports returning gain and saturation level
2559 
2560  This is used when the input exposure does not have a detector.
2561 
2562  Parameters
2563  ----------
2564  exposure : `lsst.afw.image.Exposure`
2565  Exposure to generate a fake amplifier for.
2566  config : `lsst.ip.isr.isrTaskConfig`
2567  Configuration to apply to the fake amplifier.
2568  """
2569 
2570  def __init__(self, exposure, config):
2571  self._bbox_bbox = exposure.getBBox(afwImage.LOCAL)
2572  self._RawHorizontalOverscanBBox_RawHorizontalOverscanBBox = lsst.geom.Box2I()
2573  self._gain_gain = config.gain
2574  self._readNoise_readNoise = config.readNoise
2575  self._saturation_saturation = config.saturation
2576 
2577  def getBBox(self):
2578  return self._bbox_bbox
2579 
2580  def getRawBBox(self):
2581  return self._bbox_bbox
2582 
2584  return self._RawHorizontalOverscanBBox_RawHorizontalOverscanBBox
2585 
2586  def getGain(self):
2587  return self._gain_gain
2588 
2589  def getReadNoise(self):
2590  return self._readNoise_readNoise
2591 
2592  def getSaturation(self):
2593  return self._saturation_saturation
2594 
2595  def getSuspectLevel(self):
2596  return float("NaN")
2597 
2598 
2599 class RunIsrConfig(pexConfig.Config):
2600  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2601 
2602 
2603 class RunIsrTask(pipeBase.CmdLineTask):
2604  """Task to wrap the default IsrTask to allow it to be retargeted.
2605 
2606  The standard IsrTask can be called directly from a command line
2607  program, but doing so removes the ability of the task to be
2608  retargeted. As most cameras override some set of the IsrTask
2609  methods, this would remove those data-specific methods in the
2610  output post-ISR images. This wrapping class fixes the issue,
2611  allowing identical post-ISR images to be generated by both the
2612  processCcd and isrTask code.
2613  """
2614  ConfigClass = RunIsrConfig
2615  _DefaultName = "runIsr"
2616 
2617  def __init__(self, *args, **kwargs):
2618  super().__init__(*args, **kwargs)
2619  self.makeSubtask("isr")
2620 
2621  def runDataRef(self, dataRef):
2622  """
2623  Parameters
2624  ----------
2625  dataRef : `lsst.daf.persistence.ButlerDataRef`
2626  data reference of the detector data to be processed
2627 
2628  Returns
2629  -------
2630  result : `pipeBase.Struct`
2631  Result struct with component:
2632 
2633  - exposure : `lsst.afw.image.Exposure`
2634  Post-ISR processed exposure.
2635  """
2636  return self.isr.runDataRef(dataRef)
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2583
def __init__(self, exposure, config)
Definition: isrTask.py:2570
def __init__(self, *config=None)
Definition: isrTask.py:251
def flatCorrection(self, exposure, flatExposure, invert=False)
Definition: isrTask.py:2157
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2373
def saturationInterpolation(self, exposure)
Definition: isrTask.py:2206
def runDataRef(self, sensorRef)
Definition: isrTask.py:1662
def maskNan(self, exposure)
Definition: isrTask.py:2347
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1846
def debugView(self, exposure, stepname)
Definition: isrTask.py:2535
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Definition: isrTask.py:1713
def saturationDetection(self, exposure, amp)
Definition: isrTask.py:2182
def maskDefect(self, exposure, defectBaseList)
Definition: isrTask.py:2266
def __init__(self, **kwargs)
Definition: isrTask.py:906
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:916
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
Definition: isrTask.py:2289
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1917
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2393
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2457
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2322
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Definition: isrTask.py:2481
def readIsrData(self, dataRef, rawExposure)
Definition: isrTask.py:999
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1761
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:2045
def doLinearize(self, detector)
Definition: isrTask.py:2138
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2505
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1809
def suspectDetection(self, exposure, amp)
Definition: isrTask.py:2231
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Definition: isrTask.py:1182
def darkCorrection(self, exposure, darkExposure, invert=False)
Definition: isrTask.py:2095
def __init__(self, *args, **kwargs)
Definition: isrTask.py:2617
def runDataRef(self, dataRef)
Definition: isrTask.py:2621
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
Definition: isrTask.py:62
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35