lsst.ip.isr  20.0.0-16-g111fe95+dfb030558a
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36  ReadoutCorner)
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 from lsst.meas.algorithms import Defects
43 
44 from . import isrFunctions
45 from . import isrQa
46 from . import linearize
47 
48 from .assembleCcdTask import AssembleCcdTask
49 from .crosstalk import CrosstalkTask, CrosstalkCalib
50 from .fringe import FringeTask
51 from .isr import maskNans
52 from .masking import MaskingTask
53 from .overscan import OverscanCorrectionTask
54 from .straylight import StrayLightTask
55 from .vignette import VignetteTask
56 from lsst.daf.butler import DimensionGraph
57 
58 
59 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
60 
61 
62 def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections):
63  """Lookup function to identify crosstalkSource entries.
64 
65  This should return an empty list under most circumstances. Only
66  when inter-chip crosstalk has been identified should this be
67  populated.
68 
69  This will be unused until DM-25348 resolves the quantum graph
70  generation issue.
71 
72  Parameters
73  ----------
74  datasetType : `str`
75  Dataset to lookup.
76  registry : `lsst.daf.butler.Registry`
77  Butler registry to query.
78  quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79  Data id to transform to identify crosstalkSources. The
80  ``detector`` entry will be stripped.
81  collections : `lsst.daf.butler.CollectionSearch`
82  Collections to search through.
83 
84  Returns
85  -------
86  results : `list` [`lsst.daf.butler.DatasetRef`]
87  List of datasets that match the query that will be used as
88  crosstalkSources.
89  """
90  newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"]))
91  results = list(registry.queryDatasets(datasetType,
92  collections=collections,
93  dataId=newDataId,
94  deduplicate=True,
95  ).expanded())
96  return results
97 
98 
99 class IsrTaskConnections(pipeBase.PipelineTaskConnections,
100  dimensions={"instrument", "exposure", "detector"},
101  defaultTemplates={}):
102  ccdExposure = cT.Input(
103  name="raw",
104  doc="Input exposure to process.",
105  storageClass="Exposure",
106  dimensions=["instrument", "exposure", "detector"],
107  )
108  camera = cT.PrerequisiteInput(
109  name="camera",
110  storageClass="Camera",
111  doc="Input camera to construct complete exposures.",
112  dimensions=["instrument"],
113  isCalibration=True,
114  )
115 
116  crosstalk = cT.PrerequisiteInput(
117  name="crosstalk",
118  doc="Input crosstalk object",
119  storageClass="CrosstalkCalib",
120  dimensions=["instrument", "detector"],
121  isCalibration=True,
122  )
123  # TODO: DM-25348. This does not work yet to correctly load
124  # possible crosstalk sources.
125  crosstalkSources = cT.PrerequisiteInput(
126  name="isrOverscanCorrected",
127  doc="Overscan corrected input images.",
128  storageClass="Exposure",
129  dimensions=["instrument", "exposure", "detector"],
130  deferLoad=True,
131  multiple=True,
132  lookupFunction=crosstalkSourceLookup,
133  )
134  bias = cT.PrerequisiteInput(
135  name="bias",
136  doc="Input bias calibration.",
137  storageClass="ExposureF",
138  dimensions=["instrument", "detector"],
139  isCalibration=True,
140  )
141  dark = cT.PrerequisiteInput(
142  name='dark',
143  doc="Input dark calibration.",
144  storageClass="ExposureF",
145  dimensions=["instrument", "detector"],
146  isCalibration=True,
147  )
148  flat = cT.PrerequisiteInput(
149  name="flat",
150  doc="Input flat calibration.",
151  storageClass="ExposureF",
152  dimensions=["instrument", "physical_filter", "detector"],
153  isCalibration=True,
154  )
155  fringes = cT.PrerequisiteInput(
156  name="fringe",
157  doc="Input fringe calibration.",
158  storageClass="ExposureF",
159  dimensions=["instrument", "physical_filter", "detector"],
160  isCalibration=True,
161  )
162  strayLightData = cT.PrerequisiteInput(
163  name='yBackground',
164  doc="Input stray light calibration.",
165  storageClass="StrayLightData",
166  dimensions=["instrument", "physical_filter", "detector"],
167  isCalibration=True,
168  )
169  bfKernel = cT.PrerequisiteInput(
170  name='bfKernel',
171  doc="Input brighter-fatter kernel.",
172  storageClass="NumpyArray",
173  dimensions=["instrument"],
174  isCalibration=True,
175  )
176  newBFKernel = cT.PrerequisiteInput(
177  name='brighterFatterKernel',
178  doc="Newer complete kernel + gain solutions.",
179  storageClass="BrighterFatterKernel",
180  dimensions=["instrument", "detector"],
181  isCalibration=True,
182  )
183  defects = cT.PrerequisiteInput(
184  name='defects',
185  doc="Input defect tables.",
186  storageClass="Defects",
187  dimensions=["instrument", "detector"],
188  isCalibration=True,
189  )
190  opticsTransmission = cT.PrerequisiteInput(
191  name="transmission_optics",
192  storageClass="TransmissionCurve",
193  doc="Transmission curve due to the optics.",
194  dimensions=["instrument"],
195  isCalibration=True,
196  )
197  filterTransmission = cT.PrerequisiteInput(
198  name="transmission_filter",
199  storageClass="TransmissionCurve",
200  doc="Transmission curve due to the filter.",
201  dimensions=["instrument", "physical_filter"],
202  isCalibration=True,
203  )
204  sensorTransmission = cT.PrerequisiteInput(
205  name="transmission_sensor",
206  storageClass="TransmissionCurve",
207  doc="Transmission curve due to the sensor.",
208  dimensions=["instrument", "detector"],
209  isCalibration=True,
210  )
211  atmosphereTransmission = cT.PrerequisiteInput(
212  name="transmission_atmosphere",
213  storageClass="TransmissionCurve",
214  doc="Transmission curve due to the atmosphere.",
215  dimensions=["instrument"],
216  isCalibration=True,
217  )
218  illumMaskedImage = cT.PrerequisiteInput(
219  name="illum",
220  doc="Input illumination correction.",
221  storageClass="MaskedImageF",
222  dimensions=["instrument", "physical_filter", "detector"],
223  isCalibration=True,
224  )
225 
226  outputExposure = cT.Output(
227  name='postISRCCD',
228  doc="Output ISR processed exposure.",
229  storageClass="Exposure",
230  dimensions=["instrument", "exposure", "detector"],
231  )
232  preInterpExposure = cT.Output(
233  name='preInterpISRCCD',
234  doc="Output ISR processed exposure, with pixels left uninterpolated.",
235  storageClass="ExposureF",
236  dimensions=["instrument", "exposure", "detector"],
237  )
238  outputOssThumbnail = cT.Output(
239  name="OssThumb",
240  doc="Output Overscan-subtracted thumbnail image.",
241  storageClass="Thumbnail",
242  dimensions=["instrument", "exposure", "detector"],
243  )
244  outputFlattenedThumbnail = cT.Output(
245  name="FlattenedThumb",
246  doc="Output flat-corrected thumbnail image.",
247  storageClass="Thumbnail",
248  dimensions=["instrument", "exposure", "detector"],
249  )
250 
251  def __init__(self, *, config=None):
252  super().__init__(config=config)
253 
254  if config.doBias is not True:
255  self.prerequisiteInputs.discard("bias")
256  if config.doLinearize is not True:
257  self.prerequisiteInputs.discard("linearizer")
258  if config.doCrosstalk is not True:
259  self.inputs.discard("crosstalkSources")
260  self.prerequisiteInputs.discard("crosstalk")
261  if config.doBrighterFatter is not True:
262  self.prerequisiteInputs.discard("bfKernel")
263  self.prerequisiteInputs.discard("newBFKernel")
264  if config.doDefect is not True:
265  self.prerequisiteInputs.discard("defects")
266  if config.doDark is not True:
267  self.prerequisiteInputs.discard("dark")
268  if config.doFlat is not True:
269  self.prerequisiteInputs.discard("flat")
270  if config.doAttachTransmissionCurve is not True:
271  self.prerequisiteInputs.discard("opticsTransmission")
272  self.prerequisiteInputs.discard("filterTransmission")
273  self.prerequisiteInputs.discard("sensorTransmission")
274  self.prerequisiteInputs.discard("atmosphereTransmission")
275  if config.doUseOpticsTransmission is not True:
276  self.prerequisiteInputs.discard("opticsTransmission")
277  if config.doUseFilterTransmission is not True:
278  self.prerequisiteInputs.discard("filterTransmission")
279  if config.doUseSensorTransmission is not True:
280  self.prerequisiteInputs.discard("sensorTransmission")
281  if config.doUseAtmosphereTransmission is not True:
282  self.prerequisiteInputs.discard("atmosphereTransmission")
283  if config.doIlluminationCorrection is not True:
284  self.prerequisiteInputs.discard("illumMaskedImage")
285 
286  if config.doWrite is not True:
287  self.outputs.discard("outputExposure")
288  self.outputs.discard("preInterpExposure")
289  self.outputs.discard("outputFlattenedThumbnail")
290  self.outputs.discard("outputOssThumbnail")
291  if config.doSaveInterpPixels is not True:
292  self.outputs.discard("preInterpExposure")
293  if config.qa.doThumbnailOss is not True:
294  self.outputs.discard("outputOssThumbnail")
295  if config.qa.doThumbnailFlattened is not True:
296  self.outputs.discard("outputFlattenedThumbnail")
297 
298 
299 class IsrTaskConfig(pipeBase.PipelineTaskConfig,
300  pipelineConnections=IsrTaskConnections):
301  """Configuration parameters for IsrTask.
302 
303  Items are grouped in the order in which they are executed by the task.
304  """
305  datasetType = pexConfig.Field(
306  dtype=str,
307  doc="Dataset type for input data; users will typically leave this alone, "
308  "but camera-specific ISR tasks will override it",
309  default="raw",
310  )
311 
312  fallbackFilterName = pexConfig.Field(
313  dtype=str,
314  doc="Fallback default filter name for calibrations.",
315  optional=True
316  )
317  useFallbackDate = pexConfig.Field(
318  dtype=bool,
319  doc="Pass observation date when using fallback filter.",
320  default=False,
321  )
322  expectWcs = pexConfig.Field(
323  dtype=bool,
324  default=True,
325  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
326  )
327  fwhm = pexConfig.Field(
328  dtype=float,
329  doc="FWHM of PSF in arcseconds.",
330  default=1.0,
331  )
332  qa = pexConfig.ConfigField(
333  dtype=isrQa.IsrQaConfig,
334  doc="QA related configuration options.",
335  )
336 
337  # Image conversion configuration
338  doConvertIntToFloat = pexConfig.Field(
339  dtype=bool,
340  doc="Convert integer raw images to floating point values?",
341  default=True,
342  )
343 
344  # Saturated pixel handling.
345  doSaturation = pexConfig.Field(
346  dtype=bool,
347  doc="Mask saturated pixels? NB: this is totally independent of the"
348  " interpolation option - this is ONLY setting the bits in the mask."
349  " To have them interpolated make sure doSaturationInterpolation=True",
350  default=True,
351  )
352  saturatedMaskName = pexConfig.Field(
353  dtype=str,
354  doc="Name of mask plane to use in saturation detection and interpolation",
355  default="SAT",
356  )
357  saturation = pexConfig.Field(
358  dtype=float,
359  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
360  default=float("NaN"),
361  )
362  growSaturationFootprintSize = pexConfig.Field(
363  dtype=int,
364  doc="Number of pixels by which to grow the saturation footprints",
365  default=1,
366  )
367 
368  # Suspect pixel handling.
369  doSuspect = pexConfig.Field(
370  dtype=bool,
371  doc="Mask suspect pixels?",
372  default=False,
373  )
374  suspectMaskName = pexConfig.Field(
375  dtype=str,
376  doc="Name of mask plane to use for suspect pixels",
377  default="SUSPECT",
378  )
379  numEdgeSuspect = pexConfig.Field(
380  dtype=int,
381  doc="Number of edge pixels to be flagged as untrustworthy.",
382  default=0,
383  )
384  edgeMaskLevel = pexConfig.ChoiceField(
385  dtype=str,
386  doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
387  default="DETECTOR",
388  allowed={
389  'DETECTOR': 'Mask only the edges of the full detector.',
390  'AMP': 'Mask edges of each amplifier.',
391  },
392  )
393 
394  # Initial masking options.
395  doSetBadRegions = pexConfig.Field(
396  dtype=bool,
397  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
398  default=True,
399  )
400  badStatistic = pexConfig.ChoiceField(
401  dtype=str,
402  doc="How to estimate the average value for BAD regions.",
403  default='MEANCLIP',
404  allowed={
405  "MEANCLIP": "Correct using the (clipped) mean of good data",
406  "MEDIAN": "Correct using the median of the good data",
407  },
408  )
409 
410  # Overscan subtraction configuration.
411  doOverscan = pexConfig.Field(
412  dtype=bool,
413  doc="Do overscan subtraction?",
414  default=True,
415  )
416  overscan = pexConfig.ConfigurableField(
417  target=OverscanCorrectionTask,
418  doc="Overscan subtraction task for image segments.",
419  )
420 
421  overscanFitType = pexConfig.ChoiceField(
422  dtype=str,
423  doc="The method for fitting the overscan bias level.",
424  default='MEDIAN',
425  allowed={
426  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
427  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
428  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
429  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
430  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
431  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
432  "MEAN": "Correct using the mean of the overscan region",
433  "MEANCLIP": "Correct using a clipped mean of the overscan region",
434  "MEDIAN": "Correct using the median of the overscan region",
435  "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
436  },
437  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
438  " This option will no longer be used, and will be removed after v20.")
439  )
440  overscanOrder = pexConfig.Field(
441  dtype=int,
442  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, "
443  "or number of spline knots if overscan fit type is a spline."),
444  default=1,
445  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
446  " This option will no longer be used, and will be removed after v20.")
447  )
448  overscanNumSigmaClip = pexConfig.Field(
449  dtype=float,
450  doc="Rejection threshold (sigma) for collapsing overscan before fit",
451  default=3.0,
452  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
453  " This option will no longer be used, and will be removed after v20.")
454  )
455  overscanIsInt = pexConfig.Field(
456  dtype=bool,
457  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
458  " and overscan.FitType=MEDIAN_PER_ROW.",
459  default=True,
460  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
461  " This option will no longer be used, and will be removed after v20.")
462  )
463  # These options do not get deprecated, as they define how we slice up the image data.
464  overscanNumLeadingColumnsToSkip = pexConfig.Field(
465  dtype=int,
466  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
467  default=0,
468  )
469  overscanNumTrailingColumnsToSkip = pexConfig.Field(
470  dtype=int,
471  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
472  default=0,
473  )
474  overscanMaxDev = pexConfig.Field(
475  dtype=float,
476  doc="Maximum deviation from the median for overscan",
477  default=1000.0, check=lambda x: x > 0
478  )
479  overscanBiasJump = pexConfig.Field(
480  dtype=bool,
481  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
482  default=False,
483  )
484  overscanBiasJumpKeyword = pexConfig.Field(
485  dtype=str,
486  doc="Header keyword containing information about devices.",
487  default="NO_SUCH_KEY",
488  )
489  overscanBiasJumpDevices = pexConfig.ListField(
490  dtype=str,
491  doc="List of devices that need piecewise overscan correction.",
492  default=(),
493  )
494  overscanBiasJumpLocation = pexConfig.Field(
495  dtype=int,
496  doc="Location of bias jump along y-axis.",
497  default=0,
498  )
499 
500  # Amplifier to CCD assembly configuration
501  doAssembleCcd = pexConfig.Field(
502  dtype=bool,
503  default=True,
504  doc="Assemble amp-level exposures into a ccd-level exposure?"
505  )
506  assembleCcd = pexConfig.ConfigurableField(
507  target=AssembleCcdTask,
508  doc="CCD assembly task",
509  )
510 
511  # General calibration configuration.
512  doAssembleIsrExposures = pexConfig.Field(
513  dtype=bool,
514  default=False,
515  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
516  )
517  doTrimToMatchCalib = pexConfig.Field(
518  dtype=bool,
519  default=False,
520  doc="Trim raw data to match calibration bounding boxes?"
521  )
522 
523  # Bias subtraction.
524  doBias = pexConfig.Field(
525  dtype=bool,
526  doc="Apply bias frame correction?",
527  default=True,
528  )
529  biasDataProductName = pexConfig.Field(
530  dtype=str,
531  doc="Name of the bias data product",
532  default="bias",
533  )
534  doBiasBeforeOverscan = pexConfig.Field(
535  dtype=bool,
536  doc="Reverse order of overscan and bias correction.",
537  default=False
538  )
539 
540  # Variance construction
541  doVariance = pexConfig.Field(
542  dtype=bool,
543  doc="Calculate variance?",
544  default=True
545  )
546  gain = pexConfig.Field(
547  dtype=float,
548  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
549  default=float("NaN"),
550  )
551  readNoise = pexConfig.Field(
552  dtype=float,
553  doc="The read noise to use if no Detector is present in the Exposure",
554  default=0.0,
555  )
556  doEmpiricalReadNoise = pexConfig.Field(
557  dtype=bool,
558  default=False,
559  doc="Calculate empirical read noise instead of value from AmpInfo data?"
560  )
561 
562  # Linearization.
563  doLinearize = pexConfig.Field(
564  dtype=bool,
565  doc="Correct for nonlinearity of the detector's response?",
566  default=True,
567  )
568 
569  # Crosstalk.
570  doCrosstalk = pexConfig.Field(
571  dtype=bool,
572  doc="Apply intra-CCD crosstalk correction?",
573  default=False,
574  )
575  doCrosstalkBeforeAssemble = pexConfig.Field(
576  dtype=bool,
577  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
578  default=False,
579  )
580  crosstalk = pexConfig.ConfigurableField(
581  target=CrosstalkTask,
582  doc="Intra-CCD crosstalk correction",
583  )
584 
585  # Masking options.
586  doDefect = pexConfig.Field(
587  dtype=bool,
588  doc="Apply correction for CCD defects, e.g. hot pixels?",
589  default=True,
590  )
591  doNanMasking = pexConfig.Field(
592  dtype=bool,
593  doc="Mask NAN pixels?",
594  default=True,
595  )
596  doWidenSaturationTrails = pexConfig.Field(
597  dtype=bool,
598  doc="Widen bleed trails based on their width?",
599  default=True
600  )
601 
602  # Brighter-Fatter correction.
603  doBrighterFatter = pexConfig.Field(
604  dtype=bool,
605  default=False,
606  doc="Apply the brighter fatter correction"
607  )
608  brighterFatterLevel = pexConfig.ChoiceField(
609  dtype=str,
610  default="DETECTOR",
611  doc="The level at which to correct for brighter-fatter.",
612  allowed={
613  "AMP": "Every amplifier treated separately.",
614  "DETECTOR": "One kernel per detector",
615  }
616  )
617  brighterFatterMaxIter = pexConfig.Field(
618  dtype=int,
619  default=10,
620  doc="Maximum number of iterations for the brighter fatter correction"
621  )
622  brighterFatterThreshold = pexConfig.Field(
623  dtype=float,
624  default=1000,
625  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
626  " absolute value of the difference between the current corrected image and the one"
627  " from the previous iteration summed over all the pixels."
628  )
629  brighterFatterApplyGain = pexConfig.Field(
630  dtype=bool,
631  default=True,
632  doc="Should the gain be applied when applying the brighter fatter correction?"
633  )
634  brighterFatterMaskGrowSize = pexConfig.Field(
635  dtype=int,
636  default=0,
637  doc="Number of pixels to grow the masks listed in config.maskListToInterpolate "
638  " when brighter-fatter correction is applied."
639  )
640 
641  # Dark subtraction.
642  doDark = pexConfig.Field(
643  dtype=bool,
644  doc="Apply dark frame correction?",
645  default=True,
646  )
647  darkDataProductName = pexConfig.Field(
648  dtype=str,
649  doc="Name of the dark data product",
650  default="dark",
651  )
652 
653  # Camera-specific stray light removal.
654  doStrayLight = pexConfig.Field(
655  dtype=bool,
656  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
657  default=False,
658  )
659  strayLight = pexConfig.ConfigurableField(
660  target=StrayLightTask,
661  doc="y-band stray light correction"
662  )
663 
664  # Flat correction.
665  doFlat = pexConfig.Field(
666  dtype=bool,
667  doc="Apply flat field correction?",
668  default=True,
669  )
670  flatDataProductName = pexConfig.Field(
671  dtype=str,
672  doc="Name of the flat data product",
673  default="flat",
674  )
675  flatScalingType = pexConfig.ChoiceField(
676  dtype=str,
677  doc="The method for scaling the flat on the fly.",
678  default='USER',
679  allowed={
680  "USER": "Scale by flatUserScale",
681  "MEAN": "Scale by the inverse of the mean",
682  "MEDIAN": "Scale by the inverse of the median",
683  },
684  )
685  flatUserScale = pexConfig.Field(
686  dtype=float,
687  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
688  default=1.0,
689  )
690  doTweakFlat = pexConfig.Field(
691  dtype=bool,
692  doc="Tweak flats to match observed amplifier ratios?",
693  default=False
694  )
695 
696  # Amplifier normalization based on gains instead of using flats configuration.
697  doApplyGains = pexConfig.Field(
698  dtype=bool,
699  doc="Correct the amplifiers for their gains instead of applying flat correction",
700  default=False,
701  )
702  normalizeGains = pexConfig.Field(
703  dtype=bool,
704  doc="Normalize all the amplifiers in each CCD to have the same median value.",
705  default=False,
706  )
707 
708  # Fringe correction.
709  doFringe = pexConfig.Field(
710  dtype=bool,
711  doc="Apply fringe correction?",
712  default=True,
713  )
714  fringe = pexConfig.ConfigurableField(
715  target=FringeTask,
716  doc="Fringe subtraction task",
717  )
718  fringeAfterFlat = pexConfig.Field(
719  dtype=bool,
720  doc="Do fringe subtraction after flat-fielding?",
721  default=True,
722  )
723 
724  # Initial CCD-level background statistics options.
725  doMeasureBackground = pexConfig.Field(
726  dtype=bool,
727  doc="Measure the background level on the reduced image?",
728  default=False,
729  )
730 
731  # Camera-specific masking configuration.
732  doCameraSpecificMasking = pexConfig.Field(
733  dtype=bool,
734  doc="Mask camera-specific bad regions?",
735  default=False,
736  )
737  masking = pexConfig.ConfigurableField(
738  target=MaskingTask,
739  doc="Masking task."
740  )
741 
742  # Interpolation options.
743 
744  doInterpolate = pexConfig.Field(
745  dtype=bool,
746  doc="Interpolate masked pixels?",
747  default=True,
748  )
749  doSaturationInterpolation = pexConfig.Field(
750  dtype=bool,
751  doc="Perform interpolation over pixels masked as saturated?"
752  " NB: This is independent of doSaturation; if that is False this plane"
753  " will likely be blank, resulting in a no-op here.",
754  default=True,
755  )
756  doNanInterpolation = pexConfig.Field(
757  dtype=bool,
758  doc="Perform interpolation over pixels masked as NaN?"
759  " NB: This is independent of doNanMasking; if that is False this plane"
760  " will likely be blank, resulting in a no-op here.",
761  default=True,
762  )
763  doNanInterpAfterFlat = pexConfig.Field(
764  dtype=bool,
765  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
766  "also have to interpolate them before flat-fielding."),
767  default=False,
768  )
769  maskListToInterpolate = pexConfig.ListField(
770  dtype=str,
771  doc="List of mask planes that should be interpolated.",
772  default=['SAT', 'BAD', 'UNMASKEDNAN'],
773  )
774  doSaveInterpPixels = pexConfig.Field(
775  dtype=bool,
776  doc="Save a copy of the pre-interpolated pixel values?",
777  default=False,
778  )
779 
780  # Default photometric calibration options.
781  fluxMag0T1 = pexConfig.DictField(
782  keytype=str,
783  itemtype=float,
784  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
785  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
786  ))
787  )
788  defaultFluxMag0T1 = pexConfig.Field(
789  dtype=float,
790  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
791  default=pow(10.0, 0.4*28.0)
792  )
793 
794  # Vignette correction configuration.
795  doVignette = pexConfig.Field(
796  dtype=bool,
797  doc="Apply vignetting parameters?",
798  default=False,
799  )
800  vignette = pexConfig.ConfigurableField(
801  target=VignetteTask,
802  doc="Vignetting task.",
803  )
804 
805  # Transmission curve configuration.
806  doAttachTransmissionCurve = pexConfig.Field(
807  dtype=bool,
808  default=False,
809  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
810  )
811  doUseOpticsTransmission = pexConfig.Field(
812  dtype=bool,
813  default=True,
814  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
815  )
816  doUseFilterTransmission = pexConfig.Field(
817  dtype=bool,
818  default=True,
819  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
820  )
821  doUseSensorTransmission = pexConfig.Field(
822  dtype=bool,
823  default=True,
824  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
825  )
826  doUseAtmosphereTransmission = pexConfig.Field(
827  dtype=bool,
828  default=True,
829  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
830  )
831 
832  # Illumination correction.
833  doIlluminationCorrection = pexConfig.Field(
834  dtype=bool,
835  default=False,
836  doc="Perform illumination correction?"
837  )
838  illuminationCorrectionDataProductName = pexConfig.Field(
839  dtype=str,
840  doc="Name of the illumination correction data product.",
841  default="illumcor",
842  )
843  illumScale = pexConfig.Field(
844  dtype=float,
845  doc="Scale factor for the illumination correction.",
846  default=1.0,
847  )
848  illumFilters = pexConfig.ListField(
849  dtype=str,
850  default=[],
851  doc="Only perform illumination correction for these filters."
852  )
853 
854  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
855  doWrite = pexConfig.Field(
856  dtype=bool,
857  doc="Persist postISRCCD?",
858  default=True,
859  )
860 
861  def validate(self):
862  super().validate()
863  if self.doFlat and self.doApplyGains:
864  raise ValueError("You may not specify both doFlat and doApplyGains")
865  if self.doBiasBeforeOverscan and self.doTrimToMatchCalib:
866  raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
867  if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate:
868  self.config.maskListToInterpolate.append("SAT")
869  if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
870  self.config.maskListToInterpolate.append("UNMASKEDNAN")
871 
872 
873 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
874  """Apply common instrument signature correction algorithms to a raw frame.
875 
876  The process for correcting imaging data is very similar from
877  camera to camera. This task provides a vanilla implementation of
878  doing these corrections, including the ability to turn certain
879  corrections off if they are not needed. The inputs to the primary
880  method, `run()`, are a raw exposure to be corrected and the
881  calibration data products. The raw input is a single chip sized
882  mosaic of all amps including overscans and other non-science
883  pixels. The method `runDataRef()` identifies and defines the
884  calibration data products, and is intended for use by a
885  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
886  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
887  subclassed for different camera, although the most camera specific
888  methods have been split into subtasks that can be redirected
889  appropriately.
890 
891  The __init__ method sets up the subtasks for ISR processing, using
892  the defaults from `lsst.ip.isr`.
893 
894  Parameters
895  ----------
896  args : `list`
897  Positional arguments passed to the Task constructor. None used at this time.
898  kwargs : `dict`, optional
899  Keyword arguments passed on to the Task constructor. None used at this time.
900  """
901  ConfigClass = IsrTaskConfig
902  _DefaultName = "isr"
903 
904  def __init__(self, **kwargs):
905  super().__init__(**kwargs)
906  self.makeSubtask("assembleCcd")
907  self.makeSubtask("crosstalk")
908  self.makeSubtask("strayLight")
909  self.makeSubtask("fringe")
910  self.makeSubtask("masking")
911  self.makeSubtask("overscan")
912  self.makeSubtask("vignette")
913 
914  def runQuantum(self, butlerQC, inputRefs, outputRefs):
915  inputs = butlerQC.get(inputRefs)
916 
917  try:
918  inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
919  except Exception as e:
920  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
921  (inputRefs, e))
922 
923  inputs['isGen3'] = True
924 
925  detector = inputs['ccdExposure'].getDetector()
926 
927  if self.config.doCrosstalk is True:
928  # Crosstalk sources need to be defined by the pipeline
929  # yaml if they exist.
930  if 'crosstalk' in inputs and inputs['crosstalk'] is not None:
931  if not isinstance(inputs['crosstalk'], CrosstalkCalib):
932  inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk'])
933  else:
934  coeffVector = (self.config.crosstalk.crosstalkValues
935  if self.config.crosstalk.useConfigCoefficients else None)
936  crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
937  inputs['crosstalk'] = crosstalkCalib
938  if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0:
939  if 'crosstalkSources' not in inputs:
940  self.log.warn("No crosstalkSources found for chip with interChip terms!")
941 
942  if self.doLinearize(detector) is True:
943  if 'linearizer' in inputs and isinstance(inputs['linearizer'], dict):
944  linearizer = linearize.Linearizer(detector=detector, log=self.log)
945  linearizer.fromYaml(inputs['linearizer'])
946  else:
947  linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), detector=detector,
948  log=self.log)
949  inputs['linearizer'] = linearizer
950 
951  if self.config.doDefect is True:
952  if "defects" in inputs and inputs['defects'] is not None:
953  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
954  # masking expects a list of defects defined by their bounding box
955  if not isinstance(inputs["defects"], Defects):
956  inputs["defects"] = Defects.fromTable(inputs["defects"])
957 
958  # Load the correct style of brighter fatter kernel, and repack
959  # the information as a numpy array.
960  if self.config.doBrighterFatter:
961  brighterFatterKernel = inputs.pop('newBFKernel', None)
962  if brighterFatterKernel is None:
963  brighterFatterKernel = inputs.get('bfKernel', None)
964 
965  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
966  detId = detector.getId()
967  inputs['bfGains'] = brighterFatterKernel.gain
968  # If the kernel is not an ndarray, it's the cp_pipe version
969  # so extract the kernel for this detector, or raise an error
970  if self.config.brighterFatterLevel == 'DETECTOR':
971  if brighterFatterKernel.detectorKernel:
972  inputs['bfKernel'] = brighterFatterKernel.detectorKernel[detId]
973  elif brighterFatterKernel.detectorKernelFromAmpKernels:
974  inputs['bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
975  else:
976  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
977  else:
978  # TODO DM-15631 for implementing this
979  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
980 
981  if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
982  expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
983  inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
984  expId=expId,
985  assembler=self.assembleCcd
986  if self.config.doAssembleIsrExposures else None)
987  else:
988  inputs['fringes'] = pipeBase.Struct(fringes=None)
989 
990  if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
991  if 'strayLightData' not in inputs:
992  inputs['strayLightData'] = None
993 
994  outputs = self.run(**inputs)
995  butlerQC.put(outputs, outputRefs)
996 
997  def readIsrData(self, dataRef, rawExposure):
998  """Retrieve necessary frames for instrument signature removal.
999 
1000  Pre-fetching all required ISR data products limits the IO
1001  required by the ISR. Any conflict between the calibration data
1002  available and that needed for ISR is also detected prior to
1003  doing processing, allowing it to fail quickly.
1004 
1005  Parameters
1006  ----------
1007  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1008  Butler reference of the detector data to be processed
1009  rawExposure : `afw.image.Exposure`
1010  The raw exposure that will later be corrected with the
1011  retrieved calibration data; should not be modified in this
1012  method.
1013 
1014  Returns
1015  -------
1016  result : `lsst.pipe.base.Struct`
1017  Result struct with components (which may be `None`):
1018  - ``bias``: bias calibration frame (`afw.image.Exposure`)
1019  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
1020  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1021  - ``dark``: dark calibration frame (`afw.image.Exposure`)
1022  - ``flat``: flat calibration frame (`afw.image.Exposure`)
1023  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1024  - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
1025  - ``fringes``: `lsst.pipe.base.Struct` with components:
1026  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1027  - ``seed``: random seed derived from the ccdExposureId for random
1028  number generator (`uint32`).
1029  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1030  A ``TransmissionCurve`` that represents the throughput of the optics,
1031  to be evaluated in focal-plane coordinates.
1032  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1033  A ``TransmissionCurve`` that represents the throughput of the filter
1034  itself, to be evaluated in focal-plane coordinates.
1035  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1036  A ``TransmissionCurve`` that represents the throughput of the sensor
1037  itself, to be evaluated in post-assembly trimmed detector coordinates.
1038  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1039  A ``TransmissionCurve`` that represents the throughput of the
1040  atmosphere, assumed to be spatially constant.
1041  - ``strayLightData`` : `object`
1042  An opaque object containing calibration information for
1043  stray-light correction. If `None`, no correction will be
1044  performed.
1045  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1046 
1047  Raises
1048  ------
1049  NotImplementedError :
1050  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1051  """
1052  try:
1053  dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1054  dateObs = dateObs.toPython().isoformat()
1055  except RuntimeError:
1056  self.log.warn("Unable to identify dateObs for rawExposure.")
1057  dateObs = None
1058 
1059  ccd = rawExposure.getDetector()
1060  filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
1061  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
1062  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
1063  if self.config.doBias else None)
1064  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
1065  linearizer = (dataRef.get("linearizer", immediate=True)
1066  if self.doLinearize(ccd) else None)
1067  if linearizer is not None and not isinstance(linearizer, numpy.ndarray):
1068  linearizer.log = self.log
1069  if isinstance(linearizer, numpy.ndarray):
1070  linearizer = linearize.Linearizer(table=linearizer, detector=ccd)
1071 
1072  crosstalkCalib = None
1073  if self.config.doCrosstalk:
1074  try:
1075  crosstalkCalib = dataRef.get("crosstalk", immediate=True)
1076  except NoResults:
1077  coeffVector = (self.config.crosstalk.crosstalkValues
1078  if self.config.crosstalk.useConfigCoefficients else None)
1079  crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1080  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1081  if self.config.doCrosstalk else None)
1082 
1083  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
1084  if self.config.doDark else None)
1085  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName,
1086  dateObs=dateObs)
1087  if self.config.doFlat else None)
1088 
1089  brighterFatterKernel = None
1090  brighterFatterGains = None
1091  if self.config.doBrighterFatter is True:
1092  try:
1093  # Use the new-style cp_pipe version of the kernel if it exists
1094  # If using a new-style kernel, always use the self-consistent
1095  # gains, i.e. the ones inside the kernel object itself
1096  brighterFatterKernel = dataRef.get("brighterFatterKernel")
1097  brighterFatterGains = brighterFatterKernel.gain
1098  self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded")
1099  except NoResults:
1100  try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
1101  brighterFatterKernel = dataRef.get("bfKernel")
1102  self.log.info("Old style bright-fatter kernel (np.array) loaded")
1103  except NoResults:
1104  brighterFatterKernel = None
1105  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1106  # If the kernel is not an ndarray, it's the cp_pipe version
1107  # so extract the kernel for this detector, or raise an error
1108  if self.config.brighterFatterLevel == 'DETECTOR':
1109  if brighterFatterKernel.detectorKernel:
1110  brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
1111  elif brighterFatterKernel.detectorKernelFromAmpKernels:
1112  brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
1113  else:
1114  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1115  else:
1116  # TODO DM-15631 for implementing this
1117  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1118 
1119  defectList = (dataRef.get("defects")
1120  if self.config.doDefect else None)
1121  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
1122  if self.config.doAssembleIsrExposures else None)
1123  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1124  else pipeBase.Struct(fringes=None))
1125 
1126  if self.config.doAttachTransmissionCurve:
1127  opticsTransmission = (dataRef.get("transmission_optics")
1128  if self.config.doUseOpticsTransmission else None)
1129  filterTransmission = (dataRef.get("transmission_filter")
1130  if self.config.doUseFilterTransmission else None)
1131  sensorTransmission = (dataRef.get("transmission_sensor")
1132  if self.config.doUseSensorTransmission else None)
1133  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1134  if self.config.doUseAtmosphereTransmission else None)
1135  else:
1136  opticsTransmission = None
1137  filterTransmission = None
1138  sensorTransmission = None
1139  atmosphereTransmission = None
1140 
1141  if self.config.doStrayLight:
1142  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1143  else:
1144  strayLightData = None
1145 
1146  illumMaskedImage = (self.getIsrExposure(dataRef,
1147  self.config.illuminationCorrectionDataProductName).getMaskedImage()
1148  if (self.config.doIlluminationCorrection
1149  and filterName in self.config.illumFilters)
1150  else None)
1151 
1152  # Struct should include only kwargs to run()
1153  return pipeBase.Struct(bias=biasExposure,
1154  linearizer=linearizer,
1155  crosstalk=crosstalkCalib,
1156  crosstalkSources=crosstalkSources,
1157  dark=darkExposure,
1158  flat=flatExposure,
1159  bfKernel=brighterFatterKernel,
1160  bfGains=brighterFatterGains,
1161  defects=defectList,
1162  fringes=fringeStruct,
1163  opticsTransmission=opticsTransmission,
1164  filterTransmission=filterTransmission,
1165  sensorTransmission=sensorTransmission,
1166  atmosphereTransmission=atmosphereTransmission,
1167  strayLightData=strayLightData,
1168  illumMaskedImage=illumMaskedImage
1169  )
1170 
1171  @pipeBase.timeMethod
1172  def run(self, ccdExposure, camera=None, bias=None, linearizer=None,
1173  crosstalk=None, crosstalkSources=None,
1174  dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1175  fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1176  sensorTransmission=None, atmosphereTransmission=None,
1177  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1178  isGen3=False,
1179  ):
1180  """Perform instrument signature removal on an exposure.
1181 
1182  Steps included in the ISR processing, in order performed, are:
1183  - saturation and suspect pixel masking
1184  - overscan subtraction
1185  - CCD assembly of individual amplifiers
1186  - bias subtraction
1187  - variance image construction
1188  - linearization of non-linear response
1189  - crosstalk masking
1190  - brighter-fatter correction
1191  - dark subtraction
1192  - fringe correction
1193  - stray light subtraction
1194  - flat correction
1195  - masking of known defects and camera specific features
1196  - vignette calculation
1197  - appending transmission curve and distortion model
1198 
1199  Parameters
1200  ----------
1201  ccdExposure : `lsst.afw.image.Exposure`
1202  The raw exposure that is to be run through ISR. The
1203  exposure is modified by this method.
1204  camera : `lsst.afw.cameraGeom.Camera`, optional
1205  The camera geometry for this exposure. Required if ``isGen3`` is
1206  `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1207  ``flat`` does not have an associated detector.
1208  bias : `lsst.afw.image.Exposure`, optional
1209  Bias calibration frame.
1210  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1211  Functor for linearization.
1212  crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1213  Calibration for crosstalk.
1214  crosstalkSources : `list`, optional
1215  List of possible crosstalk sources.
1216  dark : `lsst.afw.image.Exposure`, optional
1217  Dark calibration frame.
1218  flat : `lsst.afw.image.Exposure`, optional
1219  Flat calibration frame.
1220  bfKernel : `numpy.ndarray`, optional
1221  Brighter-fatter kernel.
1222  bfGains : `dict` of `float`, optional
1223  Gains used to override the detector's nominal gains for the
1224  brighter-fatter correction. A dict keyed by amplifier name for
1225  the detector in question.
1226  defects : `lsst.meas.algorithms.Defects`, optional
1227  List of defects.
1228  fringes : `lsst.pipe.base.Struct`, optional
1229  Struct containing the fringe correction data, with
1230  elements:
1231  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1232  - ``seed``: random seed derived from the ccdExposureId for random
1233  number generator (`uint32`)
1234  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1235  A ``TransmissionCurve`` that represents the throughput of the optics,
1236  to be evaluated in focal-plane coordinates.
1237  filterTransmission : `lsst.afw.image.TransmissionCurve`
1238  A ``TransmissionCurve`` that represents the throughput of the filter
1239  itself, to be evaluated in focal-plane coordinates.
1240  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1241  A ``TransmissionCurve`` that represents the throughput of the sensor
1242  itself, to be evaluated in post-assembly trimmed detector coordinates.
1243  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1244  A ``TransmissionCurve`` that represents the throughput of the
1245  atmosphere, assumed to be spatially constant.
1246  detectorNum : `int`, optional
1247  The integer number for the detector to process.
1248  isGen3 : bool, optional
1249  Flag this call to run() as using the Gen3 butler environment.
1250  strayLightData : `object`, optional
1251  Opaque object containing calibration information for stray-light
1252  correction. If `None`, no correction will be performed.
1253  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1254  Illumination correction image.
1255 
1256  Returns
1257  -------
1258  result : `lsst.pipe.base.Struct`
1259  Result struct with component:
1260  - ``exposure`` : `afw.image.Exposure`
1261  The fully ISR corrected exposure.
1262  - ``outputExposure`` : `afw.image.Exposure`
1263  An alias for `exposure`
1264  - ``ossThumb`` : `numpy.ndarray`
1265  Thumbnail image of the exposure after overscan subtraction.
1266  - ``flattenedThumb`` : `numpy.ndarray`
1267  Thumbnail image of the exposure after flat-field correction.
1268 
1269  Raises
1270  ------
1271  RuntimeError
1272  Raised if a configuration option is set to True, but the
1273  required calibration data has not been specified.
1274 
1275  Notes
1276  -----
1277  The current processed exposure can be viewed by setting the
1278  appropriate lsstDebug entries in the `debug.display`
1279  dictionary. The names of these entries correspond to some of
1280  the IsrTaskConfig Boolean options, with the value denoting the
1281  frame to use. The exposure is shown inside the matching
1282  option check and after the processing of that step has
1283  finished. The steps with debug points are:
1284 
1285  doAssembleCcd
1286  doBias
1287  doCrosstalk
1288  doBrighterFatter
1289  doDark
1290  doFringe
1291  doStrayLight
1292  doFlat
1293 
1294  In addition, setting the "postISRCCD" entry displays the
1295  exposure after all ISR processing has finished.
1296 
1297  """
1298 
1299  if isGen3 is True:
1300  # Gen3 currently cannot automatically do configuration overrides.
1301  # DM-15257 looks to discuss this issue.
1302  # Configure input exposures;
1303  if detectorNum is None:
1304  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1305 
1306  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1307  bias = self.ensureExposure(bias, camera, detectorNum)
1308  dark = self.ensureExposure(dark, camera, detectorNum)
1309  flat = self.ensureExposure(flat, camera, detectorNum)
1310  else:
1311  if isinstance(ccdExposure, ButlerDataRef):
1312  return self.runDataRef(ccdExposure)
1313 
1314  ccd = ccdExposure.getDetector()
1315  filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1316 
1317  if not ccd:
1318  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1319  ccd = [FakeAmp(ccdExposure, self.config)]
1320 
1321  # Validate Input
1322  if self.config.doBias and bias is None:
1323  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1324  if self.doLinearize(ccd) and linearizer is None:
1325  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1326  if self.config.doBrighterFatter and bfKernel is None:
1327  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1328  if self.config.doDark and dark is None:
1329  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1330  if self.config.doFlat and flat is None:
1331  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1332  if self.config.doDefect and defects is None:
1333  raise RuntimeError("Must supply defects if config.doDefect=True.")
1334  if (self.config.doFringe and filterName in self.fringe.config.filters
1335  and fringes.fringes is None):
1336  # The `fringes` object needs to be a pipeBase.Struct, as
1337  # we use it as a `dict` for the parameters of
1338  # `FringeTask.run()`. The `fringes.fringes` `list` may
1339  # not be `None` if `doFringe=True`. Otherwise, raise.
1340  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1341  if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters
1342  and illumMaskedImage is None):
1343  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1344 
1345  # Begin ISR processing.
1346  if self.config.doConvertIntToFloat:
1347  self.log.info("Converting exposure to floating point values.")
1348  ccdExposure = self.convertIntToFloat(ccdExposure)
1349 
1350  if self.config.doBias and self.config.doBiasBeforeOverscan:
1351  self.log.info("Applying bias correction.")
1352  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1353  trimToFit=self.config.doTrimToMatchCalib)
1354  self.debugView(ccdExposure, "doBias")
1355 
1356  # Amplifier level processing.
1357  overscans = []
1358  for amp in ccd:
1359  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1360  if ccdExposure.getBBox().contains(amp.getBBox()):
1361  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1362  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1363 
1364  if self.config.doOverscan and not badAmp:
1365  # Overscan correction on amp-by-amp basis.
1366  overscanResults = self.overscanCorrection(ccdExposure, amp)
1367  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1368  if overscanResults is not None and \
1369  self.config.qa is not None and self.config.qa.saveStats is True:
1370  if isinstance(overscanResults.overscanFit, float):
1371  qaMedian = overscanResults.overscanFit
1372  qaStdev = float("NaN")
1373  else:
1374  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1375  afwMath.MEDIAN | afwMath.STDEVCLIP)
1376  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1377  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1378 
1379  self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1380  self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1381  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1382  amp.getName(), qaMedian, qaStdev)
1383  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1384  else:
1385  if badAmp:
1386  self.log.warn("Amplifier %s is bad.", amp.getName())
1387  overscanResults = None
1388 
1389  overscans.append(overscanResults if overscanResults is not None else None)
1390  else:
1391  self.log.info("Skipped OSCAN for %s.", amp.getName())
1392 
1393  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1394  self.log.info("Applying crosstalk correction.")
1395  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1396  crosstalkSources=crosstalkSources)
1397  self.debugView(ccdExposure, "doCrosstalk")
1398 
1399  if self.config.doAssembleCcd:
1400  self.log.info("Assembling CCD from amplifiers.")
1401  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1402 
1403  if self.config.expectWcs and not ccdExposure.getWcs():
1404  self.log.warn("No WCS found in input exposure.")
1405  self.debugView(ccdExposure, "doAssembleCcd")
1406 
1407  ossThumb = None
1408  if self.config.qa.doThumbnailOss:
1409  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1410 
1411  if self.config.doBias and not self.config.doBiasBeforeOverscan:
1412  self.log.info("Applying bias correction.")
1413  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1414  trimToFit=self.config.doTrimToMatchCalib)
1415  self.debugView(ccdExposure, "doBias")
1416 
1417  if self.config.doVariance:
1418  for amp, overscanResults in zip(ccd, overscans):
1419  if ccdExposure.getBBox().contains(amp.getBBox()):
1420  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1421  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1422  if overscanResults is not None:
1423  self.updateVariance(ampExposure, amp,
1424  overscanImage=overscanResults.overscanImage)
1425  else:
1426  self.updateVariance(ampExposure, amp,
1427  overscanImage=None)
1428  if self.config.qa is not None and self.config.qa.saveStats is True:
1429  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1430  afwMath.MEDIAN | afwMath.STDEVCLIP)
1431  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1432  qaStats.getValue(afwMath.MEDIAN))
1433  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1434  qaStats.getValue(afwMath.STDEVCLIP))
1435  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1436  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1437  qaStats.getValue(afwMath.STDEVCLIP))
1438 
1439  if self.doLinearize(ccd):
1440  self.log.info("Applying linearizer.")
1441  linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1442  detector=ccd, log=self.log)
1443 
1444  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1445  self.log.info("Applying crosstalk correction.")
1446  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1447  crosstalkSources=crosstalkSources, isTrimmed=True)
1448  self.debugView(ccdExposure, "doCrosstalk")
1449 
1450  # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1451  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1452  if self.config.doDefect:
1453  self.log.info("Masking defects.")
1454  self.maskDefect(ccdExposure, defects)
1455 
1456  if self.config.numEdgeSuspect > 0:
1457  self.log.info("Masking edges as SUSPECT.")
1458  self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1459  maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
1460 
1461  if self.config.doNanMasking:
1462  self.log.info("Masking NAN value pixels.")
1463  self.maskNan(ccdExposure)
1464 
1465  if self.config.doWidenSaturationTrails:
1466  self.log.info("Widening saturation trails.")
1467  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1468 
1469  if self.config.doCameraSpecificMasking:
1470  self.log.info("Masking regions for camera specific reasons.")
1471  self.masking.run(ccdExposure)
1472 
1473  if self.config.doBrighterFatter:
1474  # We need to apply flats and darks before we can interpolate, and we
1475  # need to interpolate before we do B-F, but we do B-F without the
1476  # flats and darks applied so we can work in units of electrons or holes.
1477  # This context manager applies and then removes the darks and flats.
1478  #
1479  # We also do not want to interpolate values here, so operate on temporary
1480  # images so we can apply only the BF-correction and roll back the
1481  # interpolation.
1482  interpExp = ccdExposure.clone()
1483  with self.flatContext(interpExp, flat, dark):
1484  isrFunctions.interpolateFromMask(
1485  maskedImage=interpExp.getMaskedImage(),
1486  fwhm=self.config.fwhm,
1487  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1488  maskNameList=self.config.maskListToInterpolate
1489  )
1490  bfExp = interpExp.clone()
1491 
1492  self.log.info("Applying brighter fatter correction using kernel type %s / gains %s.",
1493  type(bfKernel), type(bfGains))
1494  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1495  self.config.brighterFatterMaxIter,
1496  self.config.brighterFatterThreshold,
1497  self.config.brighterFatterApplyGain,
1498  bfGains)
1499  if bfResults[1] == self.config.brighterFatterMaxIter:
1500  self.log.warn("Brighter fatter correction did not converge, final difference %f.",
1501  bfResults[0])
1502  else:
1503  self.log.info("Finished brighter fatter correction in %d iterations.",
1504  bfResults[1])
1505  image = ccdExposure.getMaskedImage().getImage()
1506  bfCorr = bfExp.getMaskedImage().getImage()
1507  bfCorr -= interpExp.getMaskedImage().getImage()
1508  image += bfCorr
1509 
1510  # Applying the brighter-fatter correction applies a
1511  # convolution to the science image. At the edges this
1512  # convolution may not have sufficient valid pixels to
1513  # produce a valid correction. Mark pixels within the size
1514  # of the brighter-fatter kernel as EDGE to warn of this
1515  # fact.
1516  self.log.info("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1517  self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1518  maskPlane="EDGE")
1519 
1520  if self.config.brighterFatterMaskGrowSize > 0:
1521  self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1522  for maskPlane in self.config.maskListToInterpolate:
1523  isrFunctions.growMasks(ccdExposure.getMask(),
1524  radius=self.config.brighterFatterMaskGrowSize,
1525  maskNameList=maskPlane,
1526  maskValue=maskPlane)
1527 
1528  self.debugView(ccdExposure, "doBrighterFatter")
1529 
1530  if self.config.doDark:
1531  self.log.info("Applying dark correction.")
1532  self.darkCorrection(ccdExposure, dark)
1533  self.debugView(ccdExposure, "doDark")
1534 
1535  if self.config.doFringe and not self.config.fringeAfterFlat:
1536  self.log.info("Applying fringe correction before flat.")
1537  self.fringe.run(ccdExposure, **fringes.getDict())
1538  self.debugView(ccdExposure, "doFringe")
1539 
1540  if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1541  self.log.info("Checking strayLight correction.")
1542  self.strayLight.run(ccdExposure, strayLightData)
1543  self.debugView(ccdExposure, "doStrayLight")
1544 
1545  if self.config.doFlat:
1546  self.log.info("Applying flat correction.")
1547  self.flatCorrection(ccdExposure, flat)
1548  self.debugView(ccdExposure, "doFlat")
1549 
1550  if self.config.doApplyGains:
1551  self.log.info("Applying gain correction instead of flat.")
1552  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1554  if self.config.doFringe and self.config.fringeAfterFlat:
1555  self.log.info("Applying fringe correction after flat.")
1556  self.fringe.run(ccdExposure, **fringes.getDict())
1557 
1558  if self.config.doVignette:
1559  self.log.info("Constructing Vignette polygon.")
1560  self.vignettePolygon = self.vignette.run(ccdExposure)
1561 
1562  if self.config.vignette.doWriteVignettePolygon:
1563  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1564 
1565  if self.config.doAttachTransmissionCurve:
1566  self.log.info("Adding transmission curves.")
1567  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1568  filterTransmission=filterTransmission,
1569  sensorTransmission=sensorTransmission,
1570  atmosphereTransmission=atmosphereTransmission)
1571 
1572  flattenedThumb = None
1573  if self.config.qa.doThumbnailFlattened:
1574  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1575 
1576  if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1577  self.log.info("Performing illumination correction.")
1578  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1579  illumMaskedImage, illumScale=self.config.illumScale,
1580  trimToFit=self.config.doTrimToMatchCalib)
1581 
1582  preInterpExp = None
1583  if self.config.doSaveInterpPixels:
1584  preInterpExp = ccdExposure.clone()
1585 
1586  # Reset and interpolate bad pixels.
1587  #
1588  # Large contiguous bad regions (which should have the BAD mask
1589  # bit set) should have their values set to the image median.
1590  # This group should include defects and bad amplifiers. As the
1591  # area covered by these defects are large, there's little
1592  # reason to expect that interpolation would provide a more
1593  # useful value.
1594  #
1595  # Smaller defects can be safely interpolated after the larger
1596  # regions have had their pixel values reset. This ensures
1597  # that the remaining defects adjacent to bad amplifiers (as an
1598  # example) do not attempt to interpolate extreme values.
1599  if self.config.doSetBadRegions:
1600  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1601  if badPixelCount > 0:
1602  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1603 
1604  if self.config.doInterpolate:
1605  self.log.info("Interpolating masked pixels.")
1606  isrFunctions.interpolateFromMask(
1607  maskedImage=ccdExposure.getMaskedImage(),
1608  fwhm=self.config.fwhm,
1609  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1610  maskNameList=list(self.config.maskListToInterpolate)
1611  )
1612 
1613  self.roughZeroPoint(ccdExposure)
1614 
1615  if self.config.doMeasureBackground:
1616  self.log.info("Measuring background level.")
1617  self.measureBackground(ccdExposure, self.config.qa)
1618 
1619  if self.config.qa is not None and self.config.qa.saveStats is True:
1620  for amp in ccd:
1621  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1622  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1623  afwMath.MEDIAN | afwMath.STDEVCLIP)
1624  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1625  qaStats.getValue(afwMath.MEDIAN))
1626  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1627  qaStats.getValue(afwMath.STDEVCLIP))
1628  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1629  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1630  qaStats.getValue(afwMath.STDEVCLIP))
1631 
1632  self.debugView(ccdExposure, "postISRCCD")
1633 
1634  return pipeBase.Struct(
1635  exposure=ccdExposure,
1636  ossThumb=ossThumb,
1637  flattenedThumb=flattenedThumb,
1638 
1639  preInterpolatedExposure=preInterpExp,
1640  outputExposure=ccdExposure,
1641  outputOssThumbnail=ossThumb,
1642  outputFlattenedThumbnail=flattenedThumb,
1643  )
1644 
1645  @pipeBase.timeMethod
1646  def runDataRef(self, sensorRef):
1647  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1648 
1649  This method contains the `CmdLineTask` interface to the ISR
1650  processing. All IO is handled here, freeing the `run()` method
1651  to manage only pixel-level calculations. The steps performed
1652  are:
1653  - Read in necessary detrending/isr/calibration data.
1654  - Process raw exposure in `run()`.
1655  - Persist the ISR-corrected exposure as "postISRCCD" if
1656  config.doWrite=True.
1657 
1658  Parameters
1659  ----------
1660  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1661  DataRef of the detector data to be processed
1662 
1663  Returns
1664  -------
1665  result : `lsst.pipe.base.Struct`
1666  Result struct with component:
1667  - ``exposure`` : `afw.image.Exposure`
1668  The fully ISR corrected exposure.
1669 
1670  Raises
1671  ------
1672  RuntimeError
1673  Raised if a configuration option is set to True, but the
1674  required calibration data does not exist.
1675 
1676  """
1677  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1678 
1679  ccdExposure = sensorRef.get(self.config.datasetType)
1680 
1681  camera = sensorRef.get("camera")
1682  isrData = self.readIsrData(sensorRef, ccdExposure)
1683 
1684  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1685 
1686  if self.config.doWrite:
1687  sensorRef.put(result.exposure, "postISRCCD")
1688  if result.preInterpolatedExposure is not None:
1689  sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1690  if result.ossThumb is not None:
1691  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1692  if result.flattenedThumb is not None:
1693  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1694 
1695  return result
1696 
1697  def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1698  """Retrieve a calibration dataset for removing instrument signature.
1699 
1700  Parameters
1701  ----------
1702 
1703  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1704  DataRef of the detector data to find calibration datasets
1705  for.
1706  datasetType : `str`
1707  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1708  dateObs : `str`, optional
1709  Date of the observation. Used to correct butler failures
1710  when using fallback filters.
1711  immediate : `Bool`
1712  If True, disable butler proxies to enable error handling
1713  within this routine.
1714 
1715  Returns
1716  -------
1717  exposure : `lsst.afw.image.Exposure`
1718  Requested calibration frame.
1719 
1720  Raises
1721  ------
1722  RuntimeError
1723  Raised if no matching calibration frame can be found.
1724  """
1725  try:
1726  exp = dataRef.get(datasetType, immediate=immediate)
1727  except Exception as exc1:
1728  if not self.config.fallbackFilterName:
1729  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1730  try:
1731  if self.config.useFallbackDate and dateObs:
1732  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1733  dateObs=dateObs, immediate=immediate)
1734  else:
1735  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1736  except Exception as exc2:
1737  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1738  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1739  self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1740 
1741  if self.config.doAssembleIsrExposures:
1742  exp = self.assembleCcd.assembleCcd(exp)
1743  return exp
1744 
1745  def ensureExposure(self, inputExp, camera, detectorNum):
1746  """Ensure that the data returned by Butler is a fully constructed exposure.
1747 
1748  ISR requires exposure-level image data for historical reasons, so if we did
1749  not recieve that from Butler, construct it from what we have, modifying the
1750  input in place.
1751 
1752  Parameters
1753  ----------
1754  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1755  `lsst.afw.image.ImageF`
1756  The input data structure obtained from Butler.
1757  camera : `lsst.afw.cameraGeom.camera`
1758  The camera associated with the image. Used to find the appropriate
1759  detector.
1760  detectorNum : `int`
1761  The detector this exposure should match.
1762 
1763  Returns
1764  -------
1765  inputExp : `lsst.afw.image.Exposure`
1766  The re-constructed exposure, with appropriate detector parameters.
1767 
1768  Raises
1769  ------
1770  TypeError
1771  Raised if the input data cannot be used to construct an exposure.
1772  """
1773  if isinstance(inputExp, afwImage.DecoratedImageU):
1774  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1775  elif isinstance(inputExp, afwImage.ImageF):
1776  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1777  elif isinstance(inputExp, afwImage.MaskedImageF):
1778  inputExp = afwImage.makeExposure(inputExp)
1779  elif isinstance(inputExp, afwImage.Exposure):
1780  pass
1781  elif inputExp is None:
1782  # Assume this will be caught by the setup if it is a problem.
1783  return inputExp
1784  else:
1785  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1786  (type(inputExp), ))
1787 
1788  if inputExp.getDetector() is None:
1789  inputExp.setDetector(camera[detectorNum])
1790 
1791  return inputExp
1792 
1793  def convertIntToFloat(self, exposure):
1794  """Convert exposure image from uint16 to float.
1795 
1796  If the exposure does not need to be converted, the input is
1797  immediately returned. For exposures that are converted to use
1798  floating point pixels, the variance is set to unity and the
1799  mask to zero.
1800 
1801  Parameters
1802  ----------
1803  exposure : `lsst.afw.image.Exposure`
1804  The raw exposure to be converted.
1805 
1806  Returns
1807  -------
1808  newexposure : `lsst.afw.image.Exposure`
1809  The input ``exposure``, converted to floating point pixels.
1810 
1811  Raises
1812  ------
1813  RuntimeError
1814  Raised if the exposure type cannot be converted to float.
1815 
1816  """
1817  if isinstance(exposure, afwImage.ExposureF):
1818  # Nothing to be done
1819  self.log.debug("Exposure already of type float.")
1820  return exposure
1821  if not hasattr(exposure, "convertF"):
1822  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1823 
1824  newexposure = exposure.convertF()
1825  newexposure.variance[:] = 1
1826  newexposure.mask[:] = 0x0
1827 
1828  return newexposure
1829 
1830  def maskAmplifier(self, ccdExposure, amp, defects):
1831  """Identify bad amplifiers, saturated and suspect pixels.
1832 
1833  Parameters
1834  ----------
1835  ccdExposure : `lsst.afw.image.Exposure`
1836  Input exposure to be masked.
1837  amp : `lsst.afw.table.AmpInfoCatalog`
1838  Catalog of parameters defining the amplifier on this
1839  exposure to mask.
1840  defects : `lsst.meas.algorithms.Defects`
1841  List of defects. Used to determine if the entire
1842  amplifier is bad.
1843 
1844  Returns
1845  -------
1846  badAmp : `Bool`
1847  If this is true, the entire amplifier area is covered by
1848  defects and unusable.
1849 
1850  """
1851  maskedImage = ccdExposure.getMaskedImage()
1852 
1853  badAmp = False
1854 
1855  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1856  # comparison with current defects definition.
1857  if defects is not None:
1858  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1859 
1860  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1861  # association with pixels in current ccdExposure).
1862  if badAmp:
1863  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1864  afwImage.PARENT)
1865  maskView = dataView.getMask()
1866  maskView |= maskView.getPlaneBitMask("BAD")
1867  del maskView
1868  return badAmp
1869 
1870  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1871  # Saturation and suspect pixels can be masked now, though.
1872  limits = dict()
1873  if self.config.doSaturation and not badAmp:
1874  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1875  if self.config.doSuspect and not badAmp:
1876  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1877  if math.isfinite(self.config.saturation):
1878  limits.update({self.config.saturatedMaskName: self.config.saturation})
1879 
1880  for maskName, maskThreshold in limits.items():
1881  if not math.isnan(maskThreshold):
1882  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1883  isrFunctions.makeThresholdMask(
1884  maskedImage=dataView,
1885  threshold=maskThreshold,
1886  growFootprints=0,
1887  maskName=maskName
1888  )
1889 
1890  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1891  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1892  afwImage.PARENT)
1893  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1894  self.config.suspectMaskName])
1895  if numpy.all(maskView.getArray() & maskVal > 0):
1896  badAmp = True
1897  maskView |= maskView.getPlaneBitMask("BAD")
1898 
1899  return badAmp
1900 
1901  def overscanCorrection(self, ccdExposure, amp):
1902  """Apply overscan correction in place.
1903 
1904  This method does initial pixel rejection of the overscan
1905  region. The overscan can also be optionally segmented to
1906  allow for discontinuous overscan responses to be fit
1907  separately. The actual overscan subtraction is performed by
1908  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1909  which is called here after the amplifier is preprocessed.
1910 
1911  Parameters
1912  ----------
1913  ccdExposure : `lsst.afw.image.Exposure`
1914  Exposure to have overscan correction performed.
1915  amp : `lsst.afw.table.AmpInfoCatalog`
1916  The amplifier to consider while correcting the overscan.
1917 
1918  Returns
1919  -------
1920  overscanResults : `lsst.pipe.base.Struct`
1921  Result struct with components:
1922  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1923  Value or fit subtracted from the amplifier image data.
1924  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1925  Value or fit subtracted from the overscan image data.
1926  - ``overscanImage`` : `lsst.afw.image.Image`
1927  Image of the overscan region with the overscan
1928  correction applied. This quantity is used to estimate
1929  the amplifier read noise empirically.
1930 
1931  Raises
1932  ------
1933  RuntimeError
1934  Raised if the ``amp`` does not contain raw pixel information.
1935 
1936  See Also
1937  --------
1938  lsst.ip.isr.isrFunctions.overscanCorrection
1939  """
1940  if amp.getRawHorizontalOverscanBBox().isEmpty():
1941  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1942  return None
1943 
1944  statControl = afwMath.StatisticsControl()
1945  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1946 
1947  # Determine the bounding boxes
1948  dataBBox = amp.getRawDataBBox()
1949  oscanBBox = amp.getRawHorizontalOverscanBBox()
1950  dx0 = 0
1951  dx1 = 0
1952 
1953  prescanBBox = amp.getRawPrescanBBox()
1954  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1955  dx0 += self.config.overscanNumLeadingColumnsToSkip
1956  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1957  else:
1958  dx0 += self.config.overscanNumTrailingColumnsToSkip
1959  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1960 
1961  # Determine if we need to work on subregions of the amplifier and overscan.
1962  imageBBoxes = []
1963  overscanBBoxes = []
1964 
1965  if ((self.config.overscanBiasJump
1966  and self.config.overscanBiasJumpLocation)
1967  and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
1968  and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1969  self.config.overscanBiasJumpDevices)):
1970  if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
1971  yLower = self.config.overscanBiasJumpLocation
1972  yUpper = dataBBox.getHeight() - yLower
1973  else:
1974  yUpper = self.config.overscanBiasJumpLocation
1975  yLower = dataBBox.getHeight() - yUpper
1976 
1977  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1978  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1979  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1980  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1981  yLower)))
1982 
1983  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1984  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1985  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1986  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1987  yUpper)))
1988  else:
1989  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1990  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1991  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1992  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1993  oscanBBox.getHeight())))
1994 
1995  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1996  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1997  ampImage = ccdExposure.maskedImage[imageBBox]
1998  overscanImage = ccdExposure.maskedImage[overscanBBox]
1999 
2000  overscanArray = overscanImage.image.array
2001  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2002  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2003  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
2004 
2005  statControl = afwMath.StatisticsControl()
2006  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2007 
2008  overscanResults = self.overscan.run(ampImage.getImage(), overscanImage)
2009 
2010  # Measure average overscan levels and record them in the metadata.
2011  levelStat = afwMath.MEDIAN
2012  sigmaStat = afwMath.STDEVCLIP
2013 
2014  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2015  self.config.qa.flatness.nIter)
2016  metadata = ccdExposure.getMetadata()
2017  ampNum = amp.getName()
2018  # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
2019  if isinstance(overscanResults.overscanFit, float):
2020  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2021  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2022  else:
2023  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2024  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2025  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2026 
2027  return overscanResults
2028 
2029  def updateVariance(self, ampExposure, amp, overscanImage=None):
2030  """Set the variance plane using the amplifier gain and read noise
2031 
2032  The read noise is calculated from the ``overscanImage`` if the
2033  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2034  the value from the amplifier data is used.
2035 
2036  Parameters
2037  ----------
2038  ampExposure : `lsst.afw.image.Exposure`
2039  Exposure to process.
2040  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2041  Amplifier detector data.
2042  overscanImage : `lsst.afw.image.MaskedImage`, optional.
2043  Image of overscan, required only for empirical read noise.
2044 
2045  See also
2046  --------
2047  lsst.ip.isr.isrFunctions.updateVariance
2048  """
2049  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2050  gain = amp.getGain()
2051 
2052  if math.isnan(gain):
2053  gain = 1.0
2054  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2055  elif gain <= 0:
2056  patchedGain = 1.0
2057  self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
2058  amp.getName(), gain, patchedGain)
2059  gain = patchedGain
2060 
2061  if self.config.doEmpiricalReadNoise and overscanImage is None:
2062  self.log.info("Overscan is none for EmpiricalReadNoise.")
2063 
2064  if self.config.doEmpiricalReadNoise and overscanImage is not None:
2065  stats = afwMath.StatisticsControl()
2066  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2067  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2068  self.log.info("Calculated empirical read noise for amp %s: %f.",
2069  amp.getName(), readNoise)
2070  else:
2071  readNoise = amp.getReadNoise()
2072 
2073  isrFunctions.updateVariance(
2074  maskedImage=ampExposure.getMaskedImage(),
2075  gain=gain,
2076  readNoise=readNoise,
2077  )
2078 
2079  def darkCorrection(self, exposure, darkExposure, invert=False):
2080  """Apply dark correction in place.
2081 
2082  Parameters
2083  ----------
2084  exposure : `lsst.afw.image.Exposure`
2085  Exposure to process.
2086  darkExposure : `lsst.afw.image.Exposure`
2087  Dark exposure of the same size as ``exposure``.
2088  invert : `Bool`, optional
2089  If True, re-add the dark to an already corrected image.
2090 
2091  Raises
2092  ------
2093  RuntimeError
2094  Raised if either ``exposure`` or ``darkExposure`` do not
2095  have their dark time defined.
2096 
2097  See Also
2098  --------
2099  lsst.ip.isr.isrFunctions.darkCorrection
2100  """
2101  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2102  if math.isnan(expScale):
2103  raise RuntimeError("Exposure darktime is NAN.")
2104  if darkExposure.getInfo().getVisitInfo() is not None \
2105  and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2106  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2107  else:
2108  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
2109  # so getDarkTime() does not exist.
2110  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2111  darkScale = 1.0
2112 
2113  isrFunctions.darkCorrection(
2114  maskedImage=exposure.getMaskedImage(),
2115  darkMaskedImage=darkExposure.getMaskedImage(),
2116  expScale=expScale,
2117  darkScale=darkScale,
2118  invert=invert,
2119  trimToFit=self.config.doTrimToMatchCalib
2120  )
2121 
2122  def doLinearize(self, detector):
2123  """Check if linearization is needed for the detector cameraGeom.
2124 
2125  Checks config.doLinearize and the linearity type of the first
2126  amplifier.
2127 
2128  Parameters
2129  ----------
2130  detector : `lsst.afw.cameraGeom.Detector`
2131  Detector to get linearity type from.
2132 
2133  Returns
2134  -------
2135  doLinearize : `Bool`
2136  If True, linearization should be performed.
2137  """
2138  return self.config.doLinearize and \
2139  detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2140 
2141  def flatCorrection(self, exposure, flatExposure, invert=False):
2142  """Apply flat correction in place.
2143 
2144  Parameters
2145  ----------
2146  exposure : `lsst.afw.image.Exposure`
2147  Exposure to process.
2148  flatExposure : `lsst.afw.image.Exposure`
2149  Flat exposure of the same size as ``exposure``.
2150  invert : `Bool`, optional
2151  If True, unflatten an already flattened image.
2152 
2153  See Also
2154  --------
2155  lsst.ip.isr.isrFunctions.flatCorrection
2156  """
2157  isrFunctions.flatCorrection(
2158  maskedImage=exposure.getMaskedImage(),
2159  flatMaskedImage=flatExposure.getMaskedImage(),
2160  scalingType=self.config.flatScalingType,
2161  userScale=self.config.flatUserScale,
2162  invert=invert,
2163  trimToFit=self.config.doTrimToMatchCalib
2164  )
2165 
2166  def saturationDetection(self, exposure, amp):
2167  """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2168 
2169  Parameters
2170  ----------
2171  exposure : `lsst.afw.image.Exposure`
2172  Exposure to process. Only the amplifier DataSec is processed.
2173  amp : `lsst.afw.table.AmpInfoCatalog`
2174  Amplifier detector data.
2175 
2176  See Also
2177  --------
2178  lsst.ip.isr.isrFunctions.makeThresholdMask
2179  """
2180  if not math.isnan(amp.getSaturation()):
2181  maskedImage = exposure.getMaskedImage()
2182  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2183  isrFunctions.makeThresholdMask(
2184  maskedImage=dataView,
2185  threshold=amp.getSaturation(),
2186  growFootprints=0,
2187  maskName=self.config.saturatedMaskName,
2188  )
2189 
2190  def saturationInterpolation(self, exposure):
2191  """Interpolate over saturated pixels, in place.
2192 
2193  This method should be called after `saturationDetection`, to
2194  ensure that the saturated pixels have been identified in the
2195  SAT mask. It should also be called after `assembleCcd`, since
2196  saturated regions may cross amplifier boundaries.
2197 
2198  Parameters
2199  ----------
2200  exposure : `lsst.afw.image.Exposure`
2201  Exposure to process.
2202 
2203  See Also
2204  --------
2205  lsst.ip.isr.isrTask.saturationDetection
2206  lsst.ip.isr.isrFunctions.interpolateFromMask
2207  """
2208  isrFunctions.interpolateFromMask(
2209  maskedImage=exposure.getMaskedImage(),
2210  fwhm=self.config.fwhm,
2211  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2212  maskNameList=list(self.config.saturatedMaskName),
2213  )
2214 
2215  def suspectDetection(self, exposure, amp):
2216  """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2217 
2218  Parameters
2219  ----------
2220  exposure : `lsst.afw.image.Exposure`
2221  Exposure to process. Only the amplifier DataSec is processed.
2222  amp : `lsst.afw.table.AmpInfoCatalog`
2223  Amplifier detector data.
2224 
2225  See Also
2226  --------
2227  lsst.ip.isr.isrFunctions.makeThresholdMask
2228 
2229  Notes
2230  -----
2231  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2232  This is intended to indicate pixels that may be affected by unknown systematics;
2233  for example if non-linearity corrections above a certain level are unstable
2234  then that would be a useful value for suspectLevel. A value of `nan` indicates
2235  that no such level exists and no pixels are to be masked as suspicious.
2236  """
2237  suspectLevel = amp.getSuspectLevel()
2238  if math.isnan(suspectLevel):
2239  return
2240 
2241  maskedImage = exposure.getMaskedImage()
2242  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2243  isrFunctions.makeThresholdMask(
2244  maskedImage=dataView,
2245  threshold=suspectLevel,
2246  growFootprints=0,
2247  maskName=self.config.suspectMaskName,
2248  )
2249 
2250  def maskDefect(self, exposure, defectBaseList):
2251  """Mask defects using mask plane "BAD", in place.
2252 
2253  Parameters
2254  ----------
2255  exposure : `lsst.afw.image.Exposure`
2256  Exposure to process.
2257  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2258  `lsst.afw.image.DefectBase`.
2259  List of defects to mask.
2260 
2261  Notes
2262  -----
2263  Call this after CCD assembly, since defects may cross amplifier boundaries.
2264  """
2265  maskedImage = exposure.getMaskedImage()
2266  if not isinstance(defectBaseList, Defects):
2267  # Promotes DefectBase to Defect
2268  defectList = Defects(defectBaseList)
2269  else:
2270  defectList = defectBaseList
2271  defectList.maskPixels(maskedImage, maskName="BAD")
2272 
2273  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2274  """Mask edge pixels with applicable mask plane.
2275 
2276  Parameters
2277  ----------
2278  exposure : `lsst.afw.image.Exposure`
2279  Exposure to process.
2280  numEdgePixels : `int`, optional
2281  Number of edge pixels to mask.
2282  maskPlane : `str`, optional
2283  Mask plane name to use.
2284  level : `str`, optional
2285  Level at which to mask edges.
2286  """
2287  maskedImage = exposure.getMaskedImage()
2288  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2289 
2290  if numEdgePixels > 0:
2291  if level == 'DETECTOR':
2292  boxes = [maskedImage.getBBox()]
2293  elif level == 'AMP':
2294  boxes = [amp.getBBox() for amp in exposure.getDetector()]
2295 
2296  for box in boxes:
2297  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2298  subImage = maskedImage[box]
2299  box.grow(-numEdgePixels)
2300  # Mask pixels outside box
2301  SourceDetectionTask.setEdgeBits(
2302  subImage,
2303  box,
2304  maskBitMask)
2305 
2306  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2307  """Mask and interpolate defects using mask plane "BAD", in place.
2308 
2309  Parameters
2310  ----------
2311  exposure : `lsst.afw.image.Exposure`
2312  Exposure to process.
2313  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2314  `lsst.afw.image.DefectBase`.
2315  List of defects to mask and interpolate.
2316 
2317  See Also
2318  --------
2319  lsst.ip.isr.isrTask.maskDefect
2320  """
2321  self.maskDefect(exposure, defectBaseList)
2322  self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2323  maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2324  isrFunctions.interpolateFromMask(
2325  maskedImage=exposure.getMaskedImage(),
2326  fwhm=self.config.fwhm,
2327  growSaturatedFootprints=0,
2328  maskNameList=["BAD"],
2329  )
2330 
2331  def maskNan(self, exposure):
2332  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2333 
2334  Parameters
2335  ----------
2336  exposure : `lsst.afw.image.Exposure`
2337  Exposure to process.
2338 
2339  Notes
2340  -----
2341  We mask over all NaNs, including those that are masked with
2342  other bits (because those may or may not be interpolated over
2343  later, and we want to remove all NaNs). Despite this
2344  behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2345  the historical name.
2346  """
2347  maskedImage = exposure.getMaskedImage()
2348 
2349  # Find and mask NaNs
2350  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2351  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2352  numNans = maskNans(maskedImage, maskVal)
2353  self.metadata.set("NUMNANS", numNans)
2354  if numNans > 0:
2355  self.log.warn("There were %d unmasked NaNs.", numNans)
2356 
2357  def maskAndInterpolateNan(self, exposure):
2358  """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2359 
2360  Parameters
2361  ----------
2362  exposure : `lsst.afw.image.Exposure`
2363  Exposure to process.
2364 
2365  See Also
2366  --------
2367  lsst.ip.isr.isrTask.maskNan
2368  """
2369  self.maskNan(exposure)
2370  isrFunctions.interpolateFromMask(
2371  maskedImage=exposure.getMaskedImage(),
2372  fwhm=self.config.fwhm,
2373  growSaturatedFootprints=0,
2374  maskNameList=["UNMASKEDNAN"],
2375  )
2376 
2377  def measureBackground(self, exposure, IsrQaConfig=None):
2378  """Measure the image background in subgrids, for quality control purposes.
2379 
2380  Parameters
2381  ----------
2382  exposure : `lsst.afw.image.Exposure`
2383  Exposure to process.
2384  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2385  Configuration object containing parameters on which background
2386  statistics and subgrids to use.
2387  """
2388  if IsrQaConfig is not None:
2389  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2390  IsrQaConfig.flatness.nIter)
2391  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2392  statsControl.setAndMask(maskVal)
2393  maskedImage = exposure.getMaskedImage()
2394  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2395  skyLevel = stats.getValue(afwMath.MEDIAN)
2396  skySigma = stats.getValue(afwMath.STDEVCLIP)
2397  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2398  metadata = exposure.getMetadata()
2399  metadata.set('SKYLEVEL', skyLevel)
2400  metadata.set('SKYSIGMA', skySigma)
2401 
2402  # calcluating flatlevel over the subgrids
2403  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2404  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2405  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2406  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2407  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2408  skyLevels = numpy.zeros((nX, nY))
2409 
2410  for j in range(nY):
2411  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2412  for i in range(nX):
2413  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2414 
2415  xLLC = xc - meshXHalf
2416  yLLC = yc - meshYHalf
2417  xURC = xc + meshXHalf - 1
2418  yURC = yc + meshYHalf - 1
2419 
2420  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2421  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2422 
2423  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2424 
2425  good = numpy.where(numpy.isfinite(skyLevels))
2426  skyMedian = numpy.median(skyLevels[good])
2427  flatness = (skyLevels[good] - skyMedian) / skyMedian
2428  flatness_rms = numpy.std(flatness)
2429  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2430 
2431  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2432  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2433  nX, nY, flatness_pp, flatness_rms)
2434 
2435  metadata.set('FLATNESS_PP', float(flatness_pp))
2436  metadata.set('FLATNESS_RMS', float(flatness_rms))
2437  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2438  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2439  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2440 
2441  def roughZeroPoint(self, exposure):
2442  """Set an approximate magnitude zero point for the exposure.
2443 
2444  Parameters
2445  ----------
2446  exposure : `lsst.afw.image.Exposure`
2447  Exposure to process.
2448  """
2449  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2450  if filterName in self.config.fluxMag0T1:
2451  fluxMag0 = self.config.fluxMag0T1[filterName]
2452  else:
2453  self.log.warn("No rough magnitude zero point set for filter %s.", filterName)
2454  fluxMag0 = self.config.defaultFluxMag0T1
2455 
2456  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2457  if not expTime > 0: # handle NaN as well as <= 0
2458  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2459  return
2460 
2461  self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2462  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2463 
2464  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2465  """Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2466 
2467  Parameters
2468  ----------
2469  ccdExposure : `lsst.afw.image.Exposure`
2470  Exposure to process.
2471  fpPolygon : `lsst.afw.geom.Polygon`
2472  Polygon in focal plane coordinates.
2473  """
2474  # Get ccd corners in focal plane coordinates
2475  ccd = ccdExposure.getDetector()
2476  fpCorners = ccd.getCorners(FOCAL_PLANE)
2477  ccdPolygon = Polygon(fpCorners)
2478 
2479  # Get intersection of ccd corners with fpPolygon
2480  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2481 
2482  # Transform back to pixel positions and build new polygon
2483  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2484  validPolygon = Polygon(ccdPoints)
2485  ccdExposure.getInfo().setValidPolygon(validPolygon)
2486 
2487  @contextmanager
2488  def flatContext(self, exp, flat, dark=None):
2489  """Context manager that applies and removes flats and darks,
2490  if the task is configured to apply them.
2491 
2492  Parameters
2493  ----------
2494  exp : `lsst.afw.image.Exposure`
2495  Exposure to process.
2496  flat : `lsst.afw.image.Exposure`
2497  Flat exposure the same size as ``exp``.
2498  dark : `lsst.afw.image.Exposure`, optional
2499  Dark exposure the same size as ``exp``.
2500 
2501  Yields
2502  ------
2503  exp : `lsst.afw.image.Exposure`
2504  The flat and dark corrected exposure.
2505  """
2506  if self.config.doDark and dark is not None:
2507  self.darkCorrection(exp, dark)
2508  if self.config.doFlat:
2509  self.flatCorrection(exp, flat)
2510  try:
2511  yield exp
2512  finally:
2513  if self.config.doFlat:
2514  self.flatCorrection(exp, flat, invert=True)
2515  if self.config.doDark and dark is not None:
2516  self.darkCorrection(exp, dark, invert=True)
2517 
2518  def debugView(self, exposure, stepname):
2519  """Utility function to examine ISR exposure at different stages.
2520 
2521  Parameters
2522  ----------
2523  exposure : `lsst.afw.image.Exposure`
2524  Exposure to view.
2525  stepname : `str`
2526  State of processing to view.
2527  """
2528  frame = getDebugFrame(self._display, stepname)
2529  if frame:
2530  display = getDisplay(frame)
2531  display.scale('asinh', 'zscale')
2532  display.mtv(exposure)
2533  prompt = "Press Enter to continue [c]... "
2534  while True:
2535  ans = input(prompt).lower()
2536  if ans in ("", "c",):
2537  break
2538 
2539 
2540 class FakeAmp(object):
2541  """A Detector-like object that supports returning gain and saturation level
2542 
2543  This is used when the input exposure does not have a detector.
2544 
2545  Parameters
2546  ----------
2547  exposure : `lsst.afw.image.Exposure`
2548  Exposure to generate a fake amplifier for.
2549  config : `lsst.ip.isr.isrTaskConfig`
2550  Configuration to apply to the fake amplifier.
2551  """
2552 
2553  def __init__(self, exposure, config):
2554  self._bbox = exposure.getBBox(afwImage.LOCAL)
2556  self._gain = config.gain
2557  self._readNoise = config.readNoise
2558  self._saturation = config.saturation
2559 
2560  def getBBox(self):
2561  return self._bbox
2562 
2563  def getRawBBox(self):
2564  return self._bbox
2565 
2567  return self._RawHorizontalOverscanBBox
2568 
2569  def getGain(self):
2570  return self._gain
2571 
2572  def getReadNoise(self):
2573  return self._readNoise
2574 
2575  def getSaturation(self):
2576  return self._saturation
2577 
2578  def getSuspectLevel(self):
2579  return float("NaN")
2580 
2581 
2582 class RunIsrConfig(pexConfig.Config):
2583  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2584 
2585 
2586 class RunIsrTask(pipeBase.CmdLineTask):
2587  """Task to wrap the default IsrTask to allow it to be retargeted.
2588 
2589  The standard IsrTask can be called directly from a command line
2590  program, but doing so removes the ability of the task to be
2591  retargeted. As most cameras override some set of the IsrTask
2592  methods, this would remove those data-specific methods in the
2593  output post-ISR images. This wrapping class fixes the issue,
2594  allowing identical post-ISR images to be generated by both the
2595  processCcd and isrTask code.
2596  """
2597  ConfigClass = RunIsrConfig
2598  _DefaultName = "runIsr"
2599 
2600  def __init__(self, *args, **kwargs):
2601  super().__init__(*args, **kwargs)
2602  self.makeSubtask("isr")
2603 
2604  def runDataRef(self, dataRef):
2605  """
2606  Parameters
2607  ----------
2608  dataRef : `lsst.daf.persistence.ButlerDataRef`
2609  data reference of the detector data to be processed
2610 
2611  Returns
2612  -------
2613  result : `pipeBase.Struct`
2614  Result struct with component:
2615 
2616  - exposure : `lsst.afw.image.Exposure`
2617  Post-ISR processed exposure.
2618  """
2619  return self.isr.runDataRef(dataRef)
lsst::ip::isr.isrTask.IsrTaskConfig.maskListToInterpolate
maskListToInterpolate
Definition: isrTask.py:769
lsst::ip::isr.isrTask.FakeAmp.getRawBBox
def getRawBBox(self)
Definition: isrTask.py:2563
lsst::afw::image
lsst::ip::isr.linearize.Linearizer
Definition: linearize.py:37
lsst::meas::algorithms::detection
lsst::ip::isr.isrTask.IsrTask.debugView
def debugView(self, exposure, stepname)
Definition: isrTask.py:2518
lsst::ip::isr.isrTask.IsrTask.maskEdges
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
Definition: isrTask.py:2273
lsst::ip::isr.isrTask.FakeAmp.getReadNoise
def getReadNoise(self)
Definition: isrTask.py:2572
lsst::ip::isr.isrTask.IsrTask.maskAndInterpolateNan
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2357
lsst::ip::isr.isrTask.IsrTaskConfig.validate
def validate(self)
Definition: isrTask.py:861
lsst::ip::isr.isrTask.IsrTaskConnections
Definition: isrTask.py:101
lsst::ip::isr.isrTask.FakeAmp.getRawHorizontalOverscanBBox
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2566
lsst::ip::isr.isrTask.FakeAmp._RawHorizontalOverscanBBox
_RawHorizontalOverscanBBox
Definition: isrTask.py:2555
lsst::ip::isr.isrTask.IsrTask.measureBackground
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2377
lsst::ip::isr.isrTask.IsrTask.maskAmplifier
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1830
lsst::ip::isr.crosstalk.CrosstalkCalib
Definition: crosstalk.py:40
lsst::ip::isr.isrTask.IsrTask.roughZeroPoint
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2441
lsst::ip::isr.isrTask.crosstalkSourceLookup
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
Definition: isrTask.py:62
lsst::ip::isr.isrTask.IsrTask.saturationDetection
def saturationDetection(self, exposure, amp)
Definition: isrTask.py:2166
lsst::ip::isr.isrTask.IsrTask.maskNan
def maskNan(self, exposure)
Definition: isrTask.py:2331
lsst::ip::isr.isrTask.IsrTaskConfig.doNanInterpolation
doNanInterpolation
Definition: isrTask.py:756
lsst::ip::isr.isrTask.IsrTask.runDataRef
def runDataRef(self, sensorRef)
Definition: isrTask.py:1646
lsst::afw::display
lsst::ip::isr.isrTask.IsrTask.suspectDetection
def suspectDetection(self, exposure, amp)
Definition: isrTask.py:2215
lsst::ip::isr.isrTask.FakeAmp.getBBox
def getBBox(self)
Definition: isrTask.py:2560
lsst::ip::isr.isrTask.FakeAmp
Definition: isrTask.py:2540
lsst::ip::isr.isrTask.IsrTask.maskDefect
def maskDefect(self, exposure, defectBaseList)
Definition: isrTask.py:2250
lsst::ip::isr.isrTask.FakeAmp._readNoise
_readNoise
Definition: isrTask.py:2557
lsst::ip::isr.isrTask.IsrTaskConfig
Definition: isrTask.py:300
lsst::ip::isr.isrTask.IsrTask.flatCorrection
def flatCorrection(self, exposure, flatExposure, invert=False)
Definition: isrTask.py:2141
lsst::ip::isr.isrQa.IsrQaConfig
Definition: isrQa.py:58
lsst::ip::isr.isrTask.FakeAmp.getSuspectLevel
def getSuspectLevel(self)
Definition: isrTask.py:2578
lsst::ip::isr.isrTask.IsrTask
Definition: isrTask.py:873
lsst::ip::isr.isrTask.IsrTask.overscanCorrection
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1901
lsst::ip::isr.isrTask.RunIsrTask
Definition: isrTask.py:2586
lsst::ip::isr.isrTask.IsrTask.doLinearize
def doLinearize(self, detector)
Definition: isrTask.py:2122
lsst::ip::isr.isrTask.IsrTaskConfig.doTrimToMatchCalib
doTrimToMatchCalib
Definition: isrTask.py:517
lsst::ip::isr.isrTask.IsrTaskConfig.doFlat
doFlat
Definition: isrTask.py:665
lsst::ip::isr.isrTask.IsrTask.__init__
def __init__(self, **kwargs)
Definition: isrTask.py:904
lsst::ip::isr::maskNans
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
lsst::ip::isr.isrTask.IsrTask.darkCorrection
def darkCorrection(self, exposure, darkExposure, invert=False)
Definition: isrTask.py:2079
lsst::ip::isr.isrTask.IsrTask.readIsrData
def readIsrData(self, dataRef, rawExposure)
Definition: isrTask.py:997
lsst::pex::config
lsst::ip::isr.isrTask.FakeAmp._gain
_gain
Definition: isrTask.py:2556
lsst::ip::isr.isrTask.IsrTask.convertIntToFloat
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1793
lsst::ip::isr.isrTask.IsrTask.flatContext
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2488
lsst::ip::isr.isrTask.RunIsrTask.__init__
def __init__(self, *args, **kwargs)
Definition: isrTask.py:2600
lsst::ip::isr.isrTask.IsrTaskConfig.doSaturationInterpolation
doSaturationInterpolation
Definition: isrTask.py:749
lsst::ip::isr.isrTask.FakeAmp._bbox
_bbox
Definition: isrTask.py:2554
lsst::daf::persistence::butler
lsst::ip::isr.isrTask.IsrTaskConfig.doApplyGains
doApplyGains
Definition: isrTask.py:697
lsst::ip::isr.isrTask.IsrTask.vignettePolygon
vignettePolygon
Definition: isrTask.py:1553
lsst::ip::isr.isrTask.IsrTask.updateVariance
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:2029
lsst::ip::isr.isrTask.IsrTask.maskAndInterpolateDefects
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2306
lsst::ip::isr.isrTask.FakeAmp._saturation
_saturation
Definition: isrTask.py:2558
lsst::ip::isr.isrTask.FakeAmp.__init__
def __init__(self, exposure, config)
Definition: isrTask.py:2553
lsst::ip::isr.isrTask.RunIsrTask.runDataRef
def runDataRef(self, dataRef)
Definition: isrTask.py:2604
lsst::ip::isr.isrTask.FakeAmp.getGain
def getGain(self)
Definition: isrTask.py:2569
lsst::geom
lsst::ip::isr.isrTask.IsrTaskConfig.doBiasBeforeOverscan
doBiasBeforeOverscan
Definition: isrTask.py:534
lsst::ip::isr.isrTask.IsrTask.ensureExposure
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1745
lsst::afw::cameraGeom
lsst::ip::isr.isrTask.IsrTask.setValidPolygonIntersect
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Definition: isrTask.py:2464
lsst::afw::math
Point< int, 2 >
lsst::geom::Box2I
lsst::ip::isr.isrTask.FakeAmp.getSaturation
def getSaturation(self)
Definition: isrTask.py:2575
lsst::ip::isr.isrTask.IsrTask.saturationInterpolation
def saturationInterpolation(self, exposure)
Definition: isrTask.py:2190
lsst::ip::isr.isrTask.IsrTask.run
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Definition: isrTask.py:1172
lsst::daf::persistence
lsst::ip::isr.isrTask.IsrTask.runQuantum
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:914
lsst::ip::isr.isrTask.IsrTask.getIsrExposure
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Definition: isrTask.py:1697
lsst::pipe::base
lsst::meas::algorithms
lsst::ip::isr.isrTask.IsrTaskConnections.__init__
def __init__(self, *config=None)
Definition: isrTask.py:251
Extent< int, 2 >
lsst::pipe::base::connectionTypes
lsst::afw::geom
lsst::ip::isr.isrTask.RunIsrConfig
Definition: isrTask.py:2582