lsst.ip.isr  22.0.1-25-g9ca8fab+d69c21ca89
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
30 import lsst.pipe.base.connectionTypes as cT
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36  ReadoutCorner)
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 from lsst.utils.timer import timeMethod
43 
44 from . import isrFunctions
45 from . import isrQa
46 from . import linearize
47 from .defects import Defects
48 
49 from .assembleCcdTask import AssembleCcdTask
50 from .crosstalk import CrosstalkTask, CrosstalkCalib
51 from .fringe import FringeTask
52 from .isr import maskNans
53 from .masking import MaskingTask
54 from .overscan import OverscanCorrectionTask
55 from .straylight import StrayLightTask
56 from .vignette import VignetteTask
57 from .ampOffset import AmpOffsetTask
58 from lsst.daf.butler import DimensionGraph
59 
60 
61 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
62 
63 
64 def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections):
65  """Lookup function to identify crosstalkSource entries.
66 
67  This should return an empty list under most circumstances. Only
68  when inter-chip crosstalk has been identified should this be
69  populated.
70 
71  Parameters
72  ----------
73  datasetType : `str`
74  Dataset to lookup.
75  registry : `lsst.daf.butler.Registry`
76  Butler registry to query.
77  quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
78  Data id to transform to identify crosstalkSources. The
79  ``detector`` entry will be stripped.
80  collections : `lsst.daf.butler.CollectionSearch`
81  Collections to search through.
82 
83  Returns
84  -------
85  results : `list` [`lsst.daf.butler.DatasetRef`]
86  List of datasets that match the query that will be used as
87  crosstalkSources.
88  """
89  newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"]))
90  results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId,
91  findFirst=True))
92  # In some contexts, calling `.expanded()` to expand all data IDs in the
93  # query results can be a lot faster because it vectorizes lookups. But in
94  # this case, expandDataId shouldn't need to hit the database at all in the
95  # steady state, because only the detector record is unknown and those are
96  # cached in the registry.
97  return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records)) for ref in results]
98 
99 
100 class IsrTaskConnections(pipeBase.PipelineTaskConnections,
101  dimensions={"instrument", "exposure", "detector"},
102  defaultTemplates={}):
103  ccdExposure = cT.Input(
104  name="raw",
105  doc="Input exposure to process.",
106  storageClass="Exposure",
107  dimensions=["instrument", "exposure", "detector"],
108  )
109  camera = cT.PrerequisiteInput(
110  name="camera",
111  storageClass="Camera",
112  doc="Input camera to construct complete exposures.",
113  dimensions=["instrument"],
114  isCalibration=True,
115  )
116 
117  crosstalk = cT.PrerequisiteInput(
118  name="crosstalk",
119  doc="Input crosstalk object",
120  storageClass="CrosstalkCalib",
121  dimensions=["instrument", "detector"],
122  isCalibration=True,
123  minimum=0, # can fall back to cameraGeom
124  )
125  crosstalkSources = cT.PrerequisiteInput(
126  name="isrOverscanCorrected",
127  doc="Overscan corrected input images.",
128  storageClass="Exposure",
129  dimensions=["instrument", "exposure", "detector"],
130  deferLoad=True,
131  multiple=True,
132  lookupFunction=crosstalkSourceLookup,
133  minimum=0, # not needed for all instruments, no config to control this
134  )
135  bias = cT.PrerequisiteInput(
136  name="bias",
137  doc="Input bias calibration.",
138  storageClass="ExposureF",
139  dimensions=["instrument", "detector"],
140  isCalibration=True,
141  )
142  dark = cT.PrerequisiteInput(
143  name='dark',
144  doc="Input dark calibration.",
145  storageClass="ExposureF",
146  dimensions=["instrument", "detector"],
147  isCalibration=True,
148  )
149  flat = cT.PrerequisiteInput(
150  name="flat",
151  doc="Input flat calibration.",
152  storageClass="ExposureF",
153  dimensions=["instrument", "physical_filter", "detector"],
154  isCalibration=True,
155  )
156  ptc = cT.PrerequisiteInput(
157  name="ptc",
158  doc="Input Photon Transfer Curve dataset",
159  storageClass="PhotonTransferCurveDataset",
160  dimensions=["instrument", "detector"],
161  isCalibration=True,
162  )
163  fringes = cT.PrerequisiteInput(
164  name="fringe",
165  doc="Input fringe calibration.",
166  storageClass="ExposureF",
167  dimensions=["instrument", "physical_filter", "detector"],
168  isCalibration=True,
169  minimum=0, # only needed for some bands, even when enabled
170  )
171  strayLightData = cT.PrerequisiteInput(
172  name='yBackground',
173  doc="Input stray light calibration.",
174  storageClass="StrayLightData",
175  dimensions=["instrument", "physical_filter", "detector"],
176  deferLoad=True,
177  isCalibration=True,
178  minimum=0, # only needed for some bands, even when enabled
179  )
180  bfKernel = cT.PrerequisiteInput(
181  name='bfKernel',
182  doc="Input brighter-fatter kernel.",
183  storageClass="NumpyArray",
184  dimensions=["instrument"],
185  isCalibration=True,
186  minimum=0, # can use either bfKernel or newBFKernel
187  )
188  newBFKernel = cT.PrerequisiteInput(
189  name='brighterFatterKernel',
190  doc="Newer complete kernel + gain solutions.",
191  storageClass="BrighterFatterKernel",
192  dimensions=["instrument", "detector"],
193  isCalibration=True,
194  minimum=0, # can use either bfKernel or newBFKernel
195  )
196  defects = cT.PrerequisiteInput(
197  name='defects',
198  doc="Input defect tables.",
199  storageClass="Defects",
200  dimensions=["instrument", "detector"],
201  isCalibration=True,
202  )
203  linearizer = cT.PrerequisiteInput(
204  name='linearizer',
205  storageClass="Linearizer",
206  doc="Linearity correction calibration.",
207  dimensions=["instrument", "detector"],
208  isCalibration=True,
209  minimum=0, # can fall back to cameraGeom
210  )
211  opticsTransmission = cT.PrerequisiteInput(
212  name="transmission_optics",
213  storageClass="TransmissionCurve",
214  doc="Transmission curve due to the optics.",
215  dimensions=["instrument"],
216  isCalibration=True,
217  )
218  filterTransmission = cT.PrerequisiteInput(
219  name="transmission_filter",
220  storageClass="TransmissionCurve",
221  doc="Transmission curve due to the filter.",
222  dimensions=["instrument", "physical_filter"],
223  isCalibration=True,
224  )
225  sensorTransmission = cT.PrerequisiteInput(
226  name="transmission_sensor",
227  storageClass="TransmissionCurve",
228  doc="Transmission curve due to the sensor.",
229  dimensions=["instrument", "detector"],
230  isCalibration=True,
231  )
232  atmosphereTransmission = cT.PrerequisiteInput(
233  name="transmission_atmosphere",
234  storageClass="TransmissionCurve",
235  doc="Transmission curve due to the atmosphere.",
236  dimensions=["instrument"],
237  isCalibration=True,
238  )
239  illumMaskedImage = cT.PrerequisiteInput(
240  name="illum",
241  doc="Input illumination correction.",
242  storageClass="MaskedImageF",
243  dimensions=["instrument", "physical_filter", "detector"],
244  isCalibration=True,
245  )
246 
247  outputExposure = cT.Output(
248  name='postISRCCD',
249  doc="Output ISR processed exposure.",
250  storageClass="Exposure",
251  dimensions=["instrument", "exposure", "detector"],
252  )
253  preInterpExposure = cT.Output(
254  name='preInterpISRCCD',
255  doc="Output ISR processed exposure, with pixels left uninterpolated.",
256  storageClass="ExposureF",
257  dimensions=["instrument", "exposure", "detector"],
258  )
259  outputOssThumbnail = cT.Output(
260  name="OssThumb",
261  doc="Output Overscan-subtracted thumbnail image.",
262  storageClass="Thumbnail",
263  dimensions=["instrument", "exposure", "detector"],
264  )
265  outputFlattenedThumbnail = cT.Output(
266  name="FlattenedThumb",
267  doc="Output flat-corrected thumbnail image.",
268  storageClass="Thumbnail",
269  dimensions=["instrument", "exposure", "detector"],
270  )
271 
272  def __init__(self, *, config=None):
273  super().__init__(config=config)
274 
275  if config.doBias is not True:
276  self.prerequisiteInputs.discard("bias")
277  if config.doLinearize is not True:
278  self.prerequisiteInputs.discard("linearizer")
279  if config.doCrosstalk is not True:
280  self.prerequisiteInputs.discard("crosstalkSources")
281  self.prerequisiteInputs.discard("crosstalk")
282  if config.doBrighterFatter is not True:
283  self.prerequisiteInputs.discard("bfKernel")
284  self.prerequisiteInputs.discard("newBFKernel")
285  if config.doDefect is not True:
286  self.prerequisiteInputs.discard("defects")
287  if config.doDark is not True:
288  self.prerequisiteInputs.discard("dark")
289  if config.doFlat is not True:
290  self.prerequisiteInputs.discard("flat")
291  if config.doFringe is not True:
292  self.prerequisiteInputs.discard("fringe")
293  if config.doStrayLight is not True:
294  self.prerequisiteInputs.discard("strayLightData")
295  if config.usePtcGains is not True and config.usePtcReadNoise is not True:
296  self.prerequisiteInputs.discard("ptc")
297  if config.doAttachTransmissionCurve is not True:
298  self.prerequisiteInputs.discard("opticsTransmission")
299  self.prerequisiteInputs.discard("filterTransmission")
300  self.prerequisiteInputs.discard("sensorTransmission")
301  self.prerequisiteInputs.discard("atmosphereTransmission")
302  if config.doUseOpticsTransmission is not True:
303  self.prerequisiteInputs.discard("opticsTransmission")
304  if config.doUseFilterTransmission is not True:
305  self.prerequisiteInputs.discard("filterTransmission")
306  if config.doUseSensorTransmission is not True:
307  self.prerequisiteInputs.discard("sensorTransmission")
308  if config.doUseAtmosphereTransmission is not True:
309  self.prerequisiteInputs.discard("atmosphereTransmission")
310  if config.doIlluminationCorrection is not True:
311  self.prerequisiteInputs.discard("illumMaskedImage")
312 
313  if config.doWrite is not True:
314  self.outputs.discard("outputExposure")
315  self.outputs.discard("preInterpExposure")
316  self.outputs.discard("outputFlattenedThumbnail")
317  self.outputs.discard("outputOssThumbnail")
318  if config.doSaveInterpPixels is not True:
319  self.outputs.discard("preInterpExposure")
320  if config.qa.doThumbnailOss is not True:
321  self.outputs.discard("outputOssThumbnail")
322  if config.qa.doThumbnailFlattened is not True:
323  self.outputs.discard("outputFlattenedThumbnail")
324 
325 
326 class IsrTaskConfig(pipeBase.PipelineTaskConfig,
327  pipelineConnections=IsrTaskConnections):
328  """Configuration parameters for IsrTask.
329 
330  Items are grouped in the order in which they are executed by the task.
331  """
332  datasetType = pexConfig.Field(
333  dtype=str,
334  doc="Dataset type for input data; users will typically leave this alone, "
335  "but camera-specific ISR tasks will override it",
336  default="raw",
337  )
338 
339  fallbackFilterName = pexConfig.Field(
340  dtype=str,
341  doc="Fallback default filter name for calibrations.",
342  optional=True
343  )
344  useFallbackDate = pexConfig.Field(
345  dtype=bool,
346  doc="Pass observation date when using fallback filter.",
347  default=False,
348  )
349  expectWcs = pexConfig.Field(
350  dtype=bool,
351  default=True,
352  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
353  )
354  fwhm = pexConfig.Field(
355  dtype=float,
356  doc="FWHM of PSF in arcseconds.",
357  default=1.0,
358  )
359  qa = pexConfig.ConfigField(
360  dtype=isrQa.IsrQaConfig,
361  doc="QA related configuration options.",
362  )
363 
364  # Image conversion configuration
365  doConvertIntToFloat = pexConfig.Field(
366  dtype=bool,
367  doc="Convert integer raw images to floating point values?",
368  default=True,
369  )
370 
371  # Saturated pixel handling.
372  doSaturation = pexConfig.Field(
373  dtype=bool,
374  doc="Mask saturated pixels? NB: this is totally independent of the"
375  " interpolation option - this is ONLY setting the bits in the mask."
376  " To have them interpolated make sure doSaturationInterpolation=True",
377  default=True,
378  )
379  saturatedMaskName = pexConfig.Field(
380  dtype=str,
381  doc="Name of mask plane to use in saturation detection and interpolation",
382  default="SAT",
383  )
384  saturation = pexConfig.Field(
385  dtype=float,
386  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
387  default=float("NaN"),
388  )
389  growSaturationFootprintSize = pexConfig.Field(
390  dtype=int,
391  doc="Number of pixels by which to grow the saturation footprints",
392  default=1,
393  )
394 
395  # Suspect pixel handling.
396  doSuspect = pexConfig.Field(
397  dtype=bool,
398  doc="Mask suspect pixels?",
399  default=False,
400  )
401  suspectMaskName = pexConfig.Field(
402  dtype=str,
403  doc="Name of mask plane to use for suspect pixels",
404  default="SUSPECT",
405  )
406  numEdgeSuspect = pexConfig.Field(
407  dtype=int,
408  doc="Number of edge pixels to be flagged as untrustworthy.",
409  default=0,
410  )
411  edgeMaskLevel = pexConfig.ChoiceField(
412  dtype=str,
413  doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
414  default="DETECTOR",
415  allowed={
416  'DETECTOR': 'Mask only the edges of the full detector.',
417  'AMP': 'Mask edges of each amplifier.',
418  },
419  )
420 
421  # Initial masking options.
422  doSetBadRegions = pexConfig.Field(
423  dtype=bool,
424  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
425  default=True,
426  )
427  badStatistic = pexConfig.ChoiceField(
428  dtype=str,
429  doc="How to estimate the average value for BAD regions.",
430  default='MEANCLIP',
431  allowed={
432  "MEANCLIP": "Correct using the (clipped) mean of good data",
433  "MEDIAN": "Correct using the median of the good data",
434  },
435  )
436 
437  # Overscan subtraction configuration.
438  doOverscan = pexConfig.Field(
439  dtype=bool,
440  doc="Do overscan subtraction?",
441  default=True,
442  )
443  overscan = pexConfig.ConfigurableField(
444  target=OverscanCorrectionTask,
445  doc="Overscan subtraction task for image segments.",
446  )
447  overscanFitType = pexConfig.ChoiceField(
448  dtype=str,
449  doc="The method for fitting the overscan bias level.",
450  default='MEDIAN',
451  allowed={
452  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
453  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
454  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
455  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
456  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
457  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
458  "MEAN": "Correct using the mean of the overscan region",
459  "MEANCLIP": "Correct using a clipped mean of the overscan region",
460  "MEDIAN": "Correct using the median of the overscan region",
461  "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
462  },
463  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
464  " This option will no longer be used, and will be removed after v20.")
465  )
466  overscanOrder = pexConfig.Field(
467  dtype=int,
468  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, "
469  "or number of spline knots if overscan fit type is a spline."),
470  default=1,
471  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
472  " This option will no longer be used, and will be removed after v20.")
473  )
474  overscanNumSigmaClip = pexConfig.Field(
475  dtype=float,
476  doc="Rejection threshold (sigma) for collapsing overscan before fit",
477  default=3.0,
478  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
479  " This option will no longer be used, and will be removed after v20.")
480  )
481  overscanIsInt = pexConfig.Field(
482  dtype=bool,
483  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
484  " and overscan.FitType=MEDIAN_PER_ROW.",
485  default=True,
486  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
487  " This option will no longer be used, and will be removed after v20.")
488  )
489  # These options do not get deprecated, as they define how we slice up the
490  # image data.
491  overscanNumLeadingColumnsToSkip = pexConfig.Field(
492  dtype=int,
493  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
494  default=0,
495  )
496  overscanNumTrailingColumnsToSkip = pexConfig.Field(
497  dtype=int,
498  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
499  default=0,
500  )
501  overscanMaxDev = pexConfig.Field(
502  dtype=float,
503  doc="Maximum deviation from the median for overscan",
504  default=1000.0, check=lambda x: x > 0
505  )
506  overscanBiasJump = pexConfig.Field(
507  dtype=bool,
508  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
509  default=False,
510  )
511  overscanBiasJumpKeyword = pexConfig.Field(
512  dtype=str,
513  doc="Header keyword containing information about devices.",
514  default="NO_SUCH_KEY",
515  )
516  overscanBiasJumpDevices = pexConfig.ListField(
517  dtype=str,
518  doc="List of devices that need piecewise overscan correction.",
519  default=(),
520  )
521  overscanBiasJumpLocation = pexConfig.Field(
522  dtype=int,
523  doc="Location of bias jump along y-axis.",
524  default=0,
525  )
526 
527  # Amplifier to CCD assembly configuration
528  doAssembleCcd = pexConfig.Field(
529  dtype=bool,
530  default=True,
531  doc="Assemble amp-level exposures into a ccd-level exposure?"
532  )
533  assembleCcd = pexConfig.ConfigurableField(
534  target=AssembleCcdTask,
535  doc="CCD assembly task",
536  )
537 
538  # General calibration configuration.
539  doAssembleIsrExposures = pexConfig.Field(
540  dtype=bool,
541  default=False,
542  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
543  )
544  doTrimToMatchCalib = pexConfig.Field(
545  dtype=bool,
546  default=False,
547  doc="Trim raw data to match calibration bounding boxes?"
548  )
549 
550  # Bias subtraction.
551  doBias = pexConfig.Field(
552  dtype=bool,
553  doc="Apply bias frame correction?",
554  default=True,
555  )
556  biasDataProductName = pexConfig.Field(
557  dtype=str,
558  doc="Name of the bias data product",
559  default="bias",
560  )
561  doBiasBeforeOverscan = pexConfig.Field(
562  dtype=bool,
563  doc="Reverse order of overscan and bias correction.",
564  default=False
565  )
566 
567  # Variance construction
568  doVariance = pexConfig.Field(
569  dtype=bool,
570  doc="Calculate variance?",
571  default=True
572  )
573  gain = pexConfig.Field(
574  dtype=float,
575  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
576  default=float("NaN"),
577  )
578  readNoise = pexConfig.Field(
579  dtype=float,
580  doc="The read noise to use if no Detector is present in the Exposure",
581  default=0.0,
582  )
583  doEmpiricalReadNoise = pexConfig.Field(
584  dtype=bool,
585  default=False,
586  doc="Calculate empirical read noise instead of value from AmpInfo data?"
587  )
588  usePtcReadNoise = pexConfig.Field(
589  dtype=bool,
590  default=False,
591  doc="Use readnoise values from the Photon Transfer Curve?"
592  )
593  maskNegativeVariance = pexConfig.Field(
594  dtype=bool,
595  default=True,
596  doc="Mask pixels that claim a negative variance? This likely indicates a failure "
597  "in the measurement of the overscan at an edge due to the data falling off faster "
598  "than the overscan model can account for it."
599  )
600  negativeVarianceMaskName = pexConfig.Field(
601  dtype=str,
602  default="BAD",
603  doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
604  )
605  # Linearization.
606  doLinearize = pexConfig.Field(
607  dtype=bool,
608  doc="Correct for nonlinearity of the detector's response?",
609  default=True,
610  )
611 
612  # Crosstalk.
613  doCrosstalk = pexConfig.Field(
614  dtype=bool,
615  doc="Apply intra-CCD crosstalk correction?",
616  default=False,
617  )
618  doCrosstalkBeforeAssemble = pexConfig.Field(
619  dtype=bool,
620  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
621  default=False,
622  )
623  crosstalk = pexConfig.ConfigurableField(
624  target=CrosstalkTask,
625  doc="Intra-CCD crosstalk correction",
626  )
627 
628  # Masking options.
629  doDefect = pexConfig.Field(
630  dtype=bool,
631  doc="Apply correction for CCD defects, e.g. hot pixels?",
632  default=True,
633  )
634  doNanMasking = pexConfig.Field(
635  dtype=bool,
636  doc="Mask non-finite (NAN, inf) pixels?",
637  default=True,
638  )
639  doWidenSaturationTrails = pexConfig.Field(
640  dtype=bool,
641  doc="Widen bleed trails based on their width?",
642  default=True
643  )
644 
645  # Brighter-Fatter correction.
646  doBrighterFatter = pexConfig.Field(
647  dtype=bool,
648  default=False,
649  doc="Apply the brighter-fatter correction?"
650  )
651  brighterFatterLevel = pexConfig.ChoiceField(
652  dtype=str,
653  default="DETECTOR",
654  doc="The level at which to correct for brighter-fatter.",
655  allowed={
656  "AMP": "Every amplifier treated separately.",
657  "DETECTOR": "One kernel per detector",
658  }
659  )
660  brighterFatterMaxIter = pexConfig.Field(
661  dtype=int,
662  default=10,
663  doc="Maximum number of iterations for the brighter-fatter correction"
664  )
665  brighterFatterThreshold = pexConfig.Field(
666  dtype=float,
667  default=1000,
668  doc="Threshold used to stop iterating the brighter-fatter correction. It is the "
669  "absolute value of the difference between the current corrected image and the one "
670  "from the previous iteration summed over all the pixels."
671  )
672  brighterFatterApplyGain = pexConfig.Field(
673  dtype=bool,
674  default=True,
675  doc="Should the gain be applied when applying the brighter-fatter correction?"
676  )
677  brighterFatterMaskListToInterpolate = pexConfig.ListField(
678  dtype=str,
679  doc="List of mask planes that should be interpolated over when applying the brighter-fatter "
680  "correction.",
681  default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"],
682  )
683  brighterFatterMaskGrowSize = pexConfig.Field(
684  dtype=int,
685  default=0,
686  doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
687  "when brighter-fatter correction is applied."
688  )
689 
690  # Dark subtraction.
691  doDark = pexConfig.Field(
692  dtype=bool,
693  doc="Apply dark frame correction?",
694  default=True,
695  )
696  darkDataProductName = pexConfig.Field(
697  dtype=str,
698  doc="Name of the dark data product",
699  default="dark",
700  )
701 
702  # Camera-specific stray light removal.
703  doStrayLight = pexConfig.Field(
704  dtype=bool,
705  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
706  default=False,
707  )
708  strayLight = pexConfig.ConfigurableField(
709  target=StrayLightTask,
710  doc="y-band stray light correction"
711  )
712 
713  # Flat correction.
714  doFlat = pexConfig.Field(
715  dtype=bool,
716  doc="Apply flat field correction?",
717  default=True,
718  )
719  flatDataProductName = pexConfig.Field(
720  dtype=str,
721  doc="Name of the flat data product",
722  default="flat",
723  )
724  flatScalingType = pexConfig.ChoiceField(
725  dtype=str,
726  doc="The method for scaling the flat on the fly.",
727  default='USER',
728  allowed={
729  "USER": "Scale by flatUserScale",
730  "MEAN": "Scale by the inverse of the mean",
731  "MEDIAN": "Scale by the inverse of the median",
732  },
733  )
734  flatUserScale = pexConfig.Field(
735  dtype=float,
736  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
737  default=1.0,
738  )
739  doTweakFlat = pexConfig.Field(
740  dtype=bool,
741  doc="Tweak flats to match observed amplifier ratios?",
742  default=False
743  )
744 
745  # Amplifier normalization based on gains instead of using flats
746  # configuration.
747  doApplyGains = pexConfig.Field(
748  dtype=bool,
749  doc="Correct the amplifiers for their gains instead of applying flat correction",
750  default=False,
751  )
752  usePtcGains = pexConfig.Field(
753  dtype=bool,
754  doc="Use the gain values from the Photon Transfer Curve?",
755  default=False,
756  )
757  normalizeGains = pexConfig.Field(
758  dtype=bool,
759  doc="Normalize all the amplifiers in each CCD to have the same median value.",
760  default=False,
761  )
762 
763  # Fringe correction.
764  doFringe = pexConfig.Field(
765  dtype=bool,
766  doc="Apply fringe correction?",
767  default=True,
768  )
769  fringe = pexConfig.ConfigurableField(
770  target=FringeTask,
771  doc="Fringe subtraction task",
772  )
773  fringeAfterFlat = pexConfig.Field(
774  dtype=bool,
775  doc="Do fringe subtraction after flat-fielding?",
776  default=True,
777  )
778 
779  # Amp offset correction.
780  doAmpOffset = pexConfig.Field(
781  doc="Calculate and apply amp offset corrections?",
782  dtype=bool,
783  default=False,
784  )
785  ampOffset = pexConfig.ConfigurableField(
786  doc="Amp offset correction task.",
787  target=AmpOffsetTask,
788  )
789 
790  # Initial CCD-level background statistics options.
791  doMeasureBackground = pexConfig.Field(
792  dtype=bool,
793  doc="Measure the background level on the reduced image?",
794  default=False,
795  )
796 
797  # Camera-specific masking configuration.
798  doCameraSpecificMasking = pexConfig.Field(
799  dtype=bool,
800  doc="Mask camera-specific bad regions?",
801  default=False,
802  )
803  masking = pexConfig.ConfigurableField(
804  target=MaskingTask,
805  doc="Masking task."
806  )
807 
808  # Interpolation options.
809  doInterpolate = pexConfig.Field(
810  dtype=bool,
811  doc="Interpolate masked pixels?",
812  default=True,
813  )
814  doSaturationInterpolation = pexConfig.Field(
815  dtype=bool,
816  doc="Perform interpolation over pixels masked as saturated?"
817  " NB: This is independent of doSaturation; if that is False this plane"
818  " will likely be blank, resulting in a no-op here.",
819  default=True,
820  )
821  doNanInterpolation = pexConfig.Field(
822  dtype=bool,
823  doc="Perform interpolation over pixels masked as NaN?"
824  " NB: This is independent of doNanMasking; if that is False this plane"
825  " will likely be blank, resulting in a no-op here.",
826  default=True,
827  )
828  doNanInterpAfterFlat = pexConfig.Field(
829  dtype=bool,
830  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
831  "also have to interpolate them before flat-fielding."),
832  default=False,
833  )
834  maskListToInterpolate = pexConfig.ListField(
835  dtype=str,
836  doc="List of mask planes that should be interpolated.",
837  default=['SAT', 'BAD'],
838  )
839  doSaveInterpPixels = pexConfig.Field(
840  dtype=bool,
841  doc="Save a copy of the pre-interpolated pixel values?",
842  default=False,
843  )
844 
845  # Default photometric calibration options.
846  fluxMag0T1 = pexConfig.DictField(
847  keytype=str,
848  itemtype=float,
849  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
850  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
851  ))
852  )
853  defaultFluxMag0T1 = pexConfig.Field(
854  dtype=float,
855  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
856  default=pow(10.0, 0.4*28.0)
857  )
858 
859  # Vignette correction configuration.
860  doVignette = pexConfig.Field(
861  dtype=bool,
862  doc="Apply vignetting parameters?",
863  default=False,
864  )
865  vignette = pexConfig.ConfigurableField(
866  target=VignetteTask,
867  doc="Vignetting task.",
868  )
869 
870  # Transmission curve configuration.
871  doAttachTransmissionCurve = pexConfig.Field(
872  dtype=bool,
873  default=False,
874  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
875  )
876  doUseOpticsTransmission = pexConfig.Field(
877  dtype=bool,
878  default=True,
879  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
880  )
881  doUseFilterTransmission = pexConfig.Field(
882  dtype=bool,
883  default=True,
884  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
885  )
886  doUseSensorTransmission = pexConfig.Field(
887  dtype=bool,
888  default=True,
889  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
890  )
891  doUseAtmosphereTransmission = pexConfig.Field(
892  dtype=bool,
893  default=True,
894  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
895  )
896 
897  # Illumination correction.
898  doIlluminationCorrection = pexConfig.Field(
899  dtype=bool,
900  default=False,
901  doc="Perform illumination correction?"
902  )
903  illuminationCorrectionDataProductName = pexConfig.Field(
904  dtype=str,
905  doc="Name of the illumination correction data product.",
906  default="illumcor",
907  )
908  illumScale = pexConfig.Field(
909  dtype=float,
910  doc="Scale factor for the illumination correction.",
911  default=1.0,
912  )
913  illumFilters = pexConfig.ListField(
914  dtype=str,
915  default=[],
916  doc="Only perform illumination correction for these filters."
917  )
918 
919  # Write the outputs to disk. If ISR is run as a subtask, this may not
920  # be needed.
921  doWrite = pexConfig.Field(
922  dtype=bool,
923  doc="Persist postISRCCD?",
924  default=True,
925  )
926 
927  def validate(self):
928  super().validate()
929  if self.doFlatdoFlat and self.doApplyGainsdoApplyGains:
930  raise ValueError("You may not specify both doFlat and doApplyGains")
931  if self.doBiasBeforeOverscandoBiasBeforeOverscan and self.doTrimToMatchCalibdoTrimToMatchCalib:
932  raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
933  if self.doSaturationInterpolationdoSaturationInterpolation and self.saturatedMaskNamesaturatedMaskName not in self.maskListToInterpolatemaskListToInterpolate:
934  self.maskListToInterpolatemaskListToInterpolate.append(self.saturatedMaskNamesaturatedMaskName)
935  if not self.doSaturationInterpolationdoSaturationInterpolation and self.saturatedMaskNamesaturatedMaskName in self.maskListToInterpolatemaskListToInterpolate:
936  self.maskListToInterpolatemaskListToInterpolate.remove(self.saturatedMaskNamesaturatedMaskName)
937  if self.doNanInterpolationdoNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolatemaskListToInterpolate:
938  self.maskListToInterpolatemaskListToInterpolate.append("UNMASKEDNAN")
939 
940 
941 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
942  """Apply common instrument signature correction algorithms to a raw frame.
943 
944  The process for correcting imaging data is very similar from
945  camera to camera. This task provides a vanilla implementation of
946  doing these corrections, including the ability to turn certain
947  corrections off if they are not needed. The inputs to the primary
948  method, `run()`, are a raw exposure to be corrected and the
949  calibration data products. The raw input is a single chip sized
950  mosaic of all amps including overscans and other non-science
951  pixels. The method `runDataRef()` identifies and defines the
952  calibration data products, and is intended for use by a
953  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
954  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
955  subclassed for different camera, although the most camera specific
956  methods have been split into subtasks that can be redirected
957  appropriately.
958 
959  The __init__ method sets up the subtasks for ISR processing, using
960  the defaults from `lsst.ip.isr`.
961 
962  Parameters
963  ----------
964  args : `list`
965  Positional arguments passed to the Task constructor.
966  None used at this time.
967  kwargs : `dict`, optional
968  Keyword arguments passed on to the Task constructor.
969  None used at this time.
970  """
971  ConfigClass = IsrTaskConfig
972  _DefaultName = "isr"
973 
974  def __init__(self, **kwargs):
975  super().__init__(**kwargs)
976  self.makeSubtask("assembleCcd")
977  self.makeSubtask("crosstalk")
978  self.makeSubtask("strayLight")
979  self.makeSubtask("fringe")
980  self.makeSubtask("masking")
981  self.makeSubtask("overscan")
982  self.makeSubtask("vignette")
983  self.makeSubtask("ampOffset")
984 
985  def runQuantum(self, butlerQC, inputRefs, outputRefs):
986  inputs = butlerQC.get(inputRefs)
987 
988  try:
989  inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
990  except Exception as e:
991  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
992  (inputRefs, e))
993 
994  inputs['isGen3'] = True
995 
996  detector = inputs['ccdExposure'].getDetector()
997 
998  if self.config.doCrosstalk is True:
999  # Crosstalk sources need to be defined by the pipeline
1000  # yaml if they exist.
1001  if 'crosstalk' in inputs and inputs['crosstalk'] is not None:
1002  if not isinstance(inputs['crosstalk'], CrosstalkCalib):
1003  inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk'])
1004  else:
1005  coeffVector = (self.config.crosstalk.crosstalkValues
1006  if self.config.crosstalk.useConfigCoefficients else None)
1007  crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
1008  inputs['crosstalk'] = crosstalkCalib
1009  if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0:
1010  if 'crosstalkSources' not in inputs:
1011  self.log.warning("No crosstalkSources found for chip with interChip terms!")
1012 
1013  if self.doLinearizedoLinearize(detector) is True:
1014  if 'linearizer' in inputs:
1015  if isinstance(inputs['linearizer'], dict):
1016  linearizer = linearize.Linearizer(detector=detector, log=self.log)
1017  linearizer.fromYaml(inputs['linearizer'])
1018  self.log.warning("Dictionary linearizers will be deprecated in DM-28741.")
1019  elif isinstance(inputs['linearizer'], numpy.ndarray):
1020  linearizer = linearize.Linearizer(table=inputs.get('linearizer', None),
1021  detector=detector,
1022  log=self.log)
1023  self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.")
1024  else:
1025  linearizer = inputs['linearizer']
1026  linearizer.log = self.log
1027  inputs['linearizer'] = linearizer
1028  else:
1029  inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log)
1030  self.log.warning("Constructing linearizer from cameraGeom information.")
1031 
1032  if self.config.doDefect is True:
1033  if "defects" in inputs and inputs['defects'] is not None:
1034  # defects is loaded as a BaseCatalog with columns
1035  # x0, y0, width, height. Masking expects a list of defects
1036  # defined by their bounding box
1037  if not isinstance(inputs["defects"], Defects):
1038  inputs["defects"] = Defects.fromTable(inputs["defects"])
1039 
1040  # Load the correct style of brighter-fatter kernel, and repack
1041  # the information as a numpy array.
1042  if self.config.doBrighterFatter:
1043  brighterFatterKernel = inputs.pop('newBFKernel', None)
1044  if brighterFatterKernel is None:
1045  brighterFatterKernel = inputs.get('bfKernel', None)
1046 
1047  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1048  # This is a ISR calib kernel
1049  detName = detector.getName()
1050  level = brighterFatterKernel.level
1051 
1052  # This is expected to be a dictionary of amp-wise gains.
1053  inputs['bfGains'] = brighterFatterKernel.gain
1054  if self.config.brighterFatterLevel == 'DETECTOR':
1055  if level == 'DETECTOR':
1056  if detName in brighterFatterKernel.detKernels:
1057  inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1058  else:
1059  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1060  elif level == 'AMP':
1061  self.log.warning("Making DETECTOR level kernel from AMP based brighter "
1062  "fatter kernels.")
1063  brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1064  inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1065  elif self.config.brighterFatterLevel == 'AMP':
1066  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1067 
1068  if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
1069  expId = inputs['ccdExposure'].info.id
1070  inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
1071  expId=expId,
1072  assembler=self.assembleCcd
1073  if self.config.doAssembleIsrExposures else None)
1074  else:
1075  inputs['fringes'] = pipeBase.Struct(fringes=None)
1076 
1077  if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
1078  if 'strayLightData' not in inputs:
1079  inputs['strayLightData'] = None
1080 
1081  outputs = self.runrun(**inputs)
1082  butlerQC.put(outputs, outputRefs)
1083 
1084  def readIsrData(self, dataRef, rawExposure):
1085  """Retrieve necessary frames for instrument signature removal.
1086 
1087  Pre-fetching all required ISR data products limits the IO
1088  required by the ISR. Any conflict between the calibration data
1089  available and that needed for ISR is also detected prior to
1090  doing processing, allowing it to fail quickly.
1091 
1092  Parameters
1093  ----------
1094  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1095  Butler reference of the detector data to be processed
1096  rawExposure : `afw.image.Exposure`
1097  The raw exposure that will later be corrected with the
1098  retrieved calibration data; should not be modified in this
1099  method.
1100 
1101  Returns
1102  -------
1103  result : `lsst.pipe.base.Struct`
1104  Result struct with components (which may be `None`):
1105  - ``bias``: bias calibration frame (`afw.image.Exposure`)
1106  - ``linearizer``: functor for linearization
1107  (`ip.isr.linearize.LinearizeBase`)
1108  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1109  - ``dark``: dark calibration frame (`afw.image.Exposure`)
1110  - ``flat``: flat calibration frame (`afw.image.Exposure`)
1111  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1112  - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1113  - ``fringes``: `lsst.pipe.base.Struct` with components:
1114  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1115  - ``seed``: random seed derived from the ccdExposureId for random
1116  number generator (`uint32`).
1117  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1118  A ``TransmissionCurve`` that represents the throughput of the
1119  optics, to be evaluated in focal-plane coordinates.
1120  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1121  A ``TransmissionCurve`` that represents the throughput of the
1122  filter itself, to be evaluated in focal-plane coordinates.
1123  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1124  A ``TransmissionCurve`` that represents the throughput of the
1125  sensor itself, to be evaluated in post-assembly trimmed
1126  detector coordinates.
1127  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1128  A ``TransmissionCurve`` that represents the throughput of the
1129  atmosphere, assumed to be spatially constant.
1130  - ``strayLightData`` : `object`
1131  An opaque object containing calibration information for
1132  stray-light correction. If `None`, no correction will be
1133  performed.
1134  - ``illumMaskedImage`` : illumination correction image
1135  (`lsst.afw.image.MaskedImage`)
1136 
1137  Raises
1138  ------
1139  NotImplementedError :
1140  Raised if a per-amplifier brighter-fatter kernel is requested by
1141  the configuration.
1142  """
1143  try:
1144  dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1145  dateObs = dateObs.toPython().isoformat()
1146  except RuntimeError:
1147  self.log.warning("Unable to identify dateObs for rawExposure.")
1148  dateObs = None
1149 
1150  ccd = rawExposure.getDetector()
1151  filterLabel = rawExposure.getFilterLabel()
1152  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1153  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
1154  biasExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.biasDataProductName)
1155  if self.config.doBias else None)
1156  # immediate=True required for functors and linearizers are functors
1157  # see ticket DM-6515
1158  linearizer = (dataRef.get("linearizer", immediate=True)
1159  if self.doLinearizedoLinearize(ccd) else None)
1160  if linearizer is not None and not isinstance(linearizer, numpy.ndarray):
1161  linearizer.log = self.log
1162  if isinstance(linearizer, numpy.ndarray):
1163  linearizer = linearize.Linearizer(table=linearizer, detector=ccd)
1164 
1165  crosstalkCalib = None
1166  if self.config.doCrosstalk:
1167  try:
1168  crosstalkCalib = dataRef.get("crosstalk", immediate=True)
1169  except NoResults:
1170  coeffVector = (self.config.crosstalk.crosstalkValues
1171  if self.config.crosstalk.useConfigCoefficients else None)
1172  crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1173  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1174  if self.config.doCrosstalk else None)
1175 
1176  darkExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.darkDataProductName)
1177  if self.config.doDark else None)
1178  flatExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.flatDataProductName,
1179  dateObs=dateObs)
1180  if self.config.doFlat else None)
1181 
1182  brighterFatterKernel = None
1183  brighterFatterGains = None
1184  if self.config.doBrighterFatter is True:
1185  try:
1186  # Use the new-style cp_pipe version of the kernel if it exists
1187  # If using a new-style kernel, always use the self-consistent
1188  # gains, i.e. the ones inside the kernel object itself
1189  brighterFatterKernel = dataRef.get("brighterFatterKernel")
1190  brighterFatterGains = brighterFatterKernel.gain
1191  self.log.info("New style brighter-fatter kernel (brighterFatterKernel) loaded")
1192  except NoResults:
1193  try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
1194  brighterFatterKernel = dataRef.get("bfKernel")
1195  self.log.info("Old style brighter-fatter kernel (bfKernel) loaded")
1196  except NoResults:
1197  brighterFatterKernel = None
1198  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1199  # If the kernel is not an ndarray, it's the cp_pipe version
1200  # so extract the kernel for this detector, or raise an error
1201  if self.config.brighterFatterLevel == 'DETECTOR':
1202  if brighterFatterKernel.detKernels:
1203  brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()]
1204  else:
1205  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1206  else:
1207  # TODO DM-15631 for implementing this
1208  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1209 
1210  defectList = (dataRef.get("defects")
1211  if self.config.doDefect else None)
1212  expId = rawExposure.info.id
1213  fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1214  if self.config.doAssembleIsrExposures else None)
1215  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1216  else pipeBase.Struct(fringes=None))
1217 
1218  if self.config.doAttachTransmissionCurve:
1219  opticsTransmission = (dataRef.get("transmission_optics")
1220  if self.config.doUseOpticsTransmission else None)
1221  filterTransmission = (dataRef.get("transmission_filter")
1222  if self.config.doUseFilterTransmission else None)
1223  sensorTransmission = (dataRef.get("transmission_sensor")
1224  if self.config.doUseSensorTransmission else None)
1225  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1226  if self.config.doUseAtmosphereTransmission else None)
1227  else:
1228  opticsTransmission = None
1229  filterTransmission = None
1230  sensorTransmission = None
1231  atmosphereTransmission = None
1232 
1233  if self.config.doStrayLight:
1234  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1235  else:
1236  strayLightData = None
1237 
1238  illumMaskedImage = (self.getIsrExposuregetIsrExposure(dataRef,
1239  self.config.illuminationCorrectionDataProductName).getMaskedImage()
1240  if (self.config.doIlluminationCorrection
1241  and physicalFilter in self.config.illumFilters)
1242  else None)
1243 
1244  # Struct should include only kwargs to run()
1245  return pipeBase.Struct(bias=biasExposure,
1246  linearizer=linearizer,
1247  crosstalk=crosstalkCalib,
1248  crosstalkSources=crosstalkSources,
1249  dark=darkExposure,
1250  flat=flatExposure,
1251  bfKernel=brighterFatterKernel,
1252  bfGains=brighterFatterGains,
1253  defects=defectList,
1254  fringes=fringeStruct,
1255  opticsTransmission=opticsTransmission,
1256  filterTransmission=filterTransmission,
1257  sensorTransmission=sensorTransmission,
1258  atmosphereTransmission=atmosphereTransmission,
1259  strayLightData=strayLightData,
1260  illumMaskedImage=illumMaskedImage
1261  )
1262 
1263  @timeMethod
1264  def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1265  crosstalk=None, crosstalkSources=None,
1266  dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1267  fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1268  sensorTransmission=None, atmosphereTransmission=None,
1269  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1270  isGen3=False,
1271  ):
1272  """Perform instrument signature removal on an exposure.
1273 
1274  Steps included in the ISR processing, in order performed, are:
1275  - saturation and suspect pixel masking
1276  - overscan subtraction
1277  - CCD assembly of individual amplifiers
1278  - bias subtraction
1279  - variance image construction
1280  - linearization of non-linear response
1281  - crosstalk masking
1282  - brighter-fatter correction
1283  - dark subtraction
1284  - fringe correction
1285  - stray light subtraction
1286  - flat correction
1287  - masking of known defects and camera specific features
1288  - vignette calculation
1289  - appending transmission curve and distortion model
1290 
1291  Parameters
1292  ----------
1293  ccdExposure : `lsst.afw.image.Exposure`
1294  The raw exposure that is to be run through ISR. The
1295  exposure is modified by this method.
1296  camera : `lsst.afw.cameraGeom.Camera`, optional
1297  The camera geometry for this exposure. Required if
1298  one or more of ``ccdExposure``, ``bias``, ``dark``, or
1299  ``flat`` does not have an associated detector.
1300  bias : `lsst.afw.image.Exposure`, optional
1301  Bias calibration frame.
1302  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1303  Functor for linearization.
1304  crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1305  Calibration for crosstalk.
1306  crosstalkSources : `list`, optional
1307  List of possible crosstalk sources.
1308  dark : `lsst.afw.image.Exposure`, optional
1309  Dark calibration frame.
1310  flat : `lsst.afw.image.Exposure`, optional
1311  Flat calibration frame.
1312  ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
1313  Photon transfer curve dataset, with, e.g., gains
1314  and read noise.
1315  bfKernel : `numpy.ndarray`, optional
1316  Brighter-fatter kernel.
1317  bfGains : `dict` of `float`, optional
1318  Gains used to override the detector's nominal gains for the
1319  brighter-fatter correction. A dict keyed by amplifier name for
1320  the detector in question.
1321  defects : `lsst.ip.isr.Defects`, optional
1322  List of defects.
1323  fringes : `lsst.pipe.base.Struct`, optional
1324  Struct containing the fringe correction data, with
1325  elements:
1326  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1327  - ``seed``: random seed derived from the ccdExposureId for random
1328  number generator (`uint32`)
1329  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1330  A ``TransmissionCurve`` that represents the throughput of the,
1331  optics, to be evaluated in focal-plane coordinates.
1332  filterTransmission : `lsst.afw.image.TransmissionCurve`
1333  A ``TransmissionCurve`` that represents the throughput of the
1334  filter itself, to be evaluated in focal-plane coordinates.
1335  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1336  A ``TransmissionCurve`` that represents the throughput of the
1337  sensor itself, to be evaluated in post-assembly trimmed detector
1338  coordinates.
1339  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1340  A ``TransmissionCurve`` that represents the throughput of the
1341  atmosphere, assumed to be spatially constant.
1342  detectorNum : `int`, optional
1343  The integer number for the detector to process.
1344  isGen3 : bool, optional
1345  Flag this call to run() as using the Gen3 butler environment.
1346  strayLightData : `object`, optional
1347  Opaque object containing calibration information for stray-light
1348  correction. If `None`, no correction will be performed.
1349  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1350  Illumination correction image.
1351 
1352  Returns
1353  -------
1354  result : `lsst.pipe.base.Struct`
1355  Result struct with component:
1356  - ``exposure`` : `afw.image.Exposure`
1357  The fully ISR corrected exposure.
1358  - ``outputExposure`` : `afw.image.Exposure`
1359  An alias for `exposure`
1360  - ``ossThumb`` : `numpy.ndarray`
1361  Thumbnail image of the exposure after overscan subtraction.
1362  - ``flattenedThumb`` : `numpy.ndarray`
1363  Thumbnail image of the exposure after flat-field correction.
1364 
1365  Raises
1366  ------
1367  RuntimeError
1368  Raised if a configuration option is set to True, but the
1369  required calibration data has not been specified.
1370 
1371  Notes
1372  -----
1373  The current processed exposure can be viewed by setting the
1374  appropriate lsstDebug entries in the `debug.display`
1375  dictionary. The names of these entries correspond to some of
1376  the IsrTaskConfig Boolean options, with the value denoting the
1377  frame to use. The exposure is shown inside the matching
1378  option check and after the processing of that step has
1379  finished. The steps with debug points are:
1380 
1381  doAssembleCcd
1382  doBias
1383  doCrosstalk
1384  doBrighterFatter
1385  doDark
1386  doFringe
1387  doStrayLight
1388  doFlat
1389 
1390  In addition, setting the "postISRCCD" entry displays the
1391  exposure after all ISR processing has finished.
1392 
1393  """
1394 
1395  if isGen3 is True:
1396  # Gen3 currently cannot automatically do configuration overrides.
1397  # DM-15257 looks to discuss this issue.
1398  # Configure input exposures;
1399 
1400  ccdExposure = self.ensureExposureensureExposure(ccdExposure, camera, detectorNum)
1401  bias = self.ensureExposureensureExposure(bias, camera, detectorNum)
1402  dark = self.ensureExposureensureExposure(dark, camera, detectorNum)
1403  flat = self.ensureExposureensureExposure(flat, camera, detectorNum)
1404  else:
1405  if isinstance(ccdExposure, ButlerDataRef):
1406  return self.runDataRefrunDataRef(ccdExposure)
1407 
1408  ccd = ccdExposure.getDetector()
1409  filterLabel = ccdExposure.getFilterLabel()
1410  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1411 
1412  if not ccd:
1413  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1414  ccd = [FakeAmp(ccdExposure, self.config)]
1415 
1416  # Validate Input
1417  if self.config.doBias and bias is None:
1418  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1419  if self.doLinearizedoLinearize(ccd) and linearizer is None:
1420  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1421  if self.config.doBrighterFatter and bfKernel is None:
1422  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1423  if self.config.doDark and dark is None:
1424  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1425  if self.config.doFlat and flat is None:
1426  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1427  if self.config.doDefect and defects is None:
1428  raise RuntimeError("Must supply defects if config.doDefect=True.")
1429  if (self.config.doFringe and physicalFilter in self.fringe.config.filters
1430  and fringes.fringes is None):
1431  # The `fringes` object needs to be a pipeBase.Struct, as
1432  # we use it as a `dict` for the parameters of
1433  # `FringeTask.run()`. The `fringes.fringes` `list` may
1434  # not be `None` if `doFringe=True`. Otherwise, raise.
1435  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1436  if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters
1437  and illumMaskedImage is None):
1438  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1439 
1440  # Begin ISR processing.
1441  if self.config.doConvertIntToFloat:
1442  self.log.info("Converting exposure to floating point values.")
1443  ccdExposure = self.convertIntToFloatconvertIntToFloat(ccdExposure)
1444 
1445  if self.config.doBias and self.config.doBiasBeforeOverscan:
1446  self.log.info("Applying bias correction.")
1447  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1448  trimToFit=self.config.doTrimToMatchCalib)
1449  self.debugViewdebugView(ccdExposure, "doBias")
1450 
1451  # Amplifier level processing.
1452  overscans = []
1453  for amp in ccd:
1454  # if ccdExposure is one amp,
1455  # check for coverage to prevent performing ops multiple times
1456  if ccdExposure.getBBox().contains(amp.getBBox()):
1457  # Check for fully masked bad amplifiers,
1458  # and generate masks for SUSPECT and SATURATED values.
1459  badAmp = self.maskAmplifiermaskAmplifier(ccdExposure, amp, defects)
1460 
1461  if self.config.doOverscan and not badAmp:
1462  # Overscan correction on amp-by-amp basis.
1463  overscanResults = self.overscanCorrectionoverscanCorrection(ccdExposure, amp)
1464  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1465  if overscanResults is not None and \
1466  self.config.qa is not None and self.config.qa.saveStats is True:
1467  if isinstance(overscanResults.overscanFit, float):
1468  qaMedian = overscanResults.overscanFit
1469  qaStdev = float("NaN")
1470  else:
1471  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1472  afwMath.MEDIAN | afwMath.STDEVCLIP)
1473  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1474  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1475 
1476  self.metadata.set(f"FIT MEDIAN {amp.getName()}", qaMedian)
1477  self.metadata.set(f"FIT STDEV {amp.getName()}", qaStdev)
1478  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1479  amp.getName(), qaMedian, qaStdev)
1480 
1481  # Residuals after overscan correction
1482  qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1483  afwMath.MEDIAN | afwMath.STDEVCLIP)
1484  qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1485  qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1486 
1487  self.metadata.set(f"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter)
1488  self.metadata.set(f"RESIDUAL STDEV {amp.getName()}", qaStdevAfter)
1489  self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f",
1490  amp.getName(), qaMedianAfter, qaStdevAfter)
1491 
1492  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1493  else:
1494  if badAmp:
1495  self.log.warning("Amplifier %s is bad.", amp.getName())
1496  overscanResults = None
1497 
1498  overscans.append(overscanResults if overscanResults is not None else None)
1499  else:
1500  self.log.info("Skipped OSCAN for %s.", amp.getName())
1501 
1502  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1503  self.log.info("Applying crosstalk correction.")
1504  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1505  crosstalkSources=crosstalkSources, camera=camera)
1506  self.debugViewdebugView(ccdExposure, "doCrosstalk")
1507 
1508  if self.config.doAssembleCcd:
1509  self.log.info("Assembling CCD from amplifiers.")
1510  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1511 
1512  if self.config.expectWcs and not ccdExposure.getWcs():
1513  self.log.warning("No WCS found in input exposure.")
1514  self.debugViewdebugView(ccdExposure, "doAssembleCcd")
1515 
1516  ossThumb = None
1517  if self.config.qa.doThumbnailOss:
1518  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1519 
1520  if self.config.doBias and not self.config.doBiasBeforeOverscan:
1521  self.log.info("Applying bias correction.")
1522  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1523  trimToFit=self.config.doTrimToMatchCalib)
1524  self.debugViewdebugView(ccdExposure, "doBias")
1525 
1526  if self.config.doVariance:
1527  for amp, overscanResults in zip(ccd, overscans):
1528  if ccdExposure.getBBox().contains(amp.getBBox()):
1529  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1530  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1531  if overscanResults is not None:
1532  self.updateVarianceupdateVariance(ampExposure, amp,
1533  overscanImage=overscanResults.overscanImage,
1534  ptcDataset=ptc)
1535  else:
1536  self.updateVarianceupdateVariance(ampExposure, amp,
1537  overscanImage=None,
1538  ptcDataset=ptc)
1539  if self.config.qa is not None and self.config.qa.saveStats is True:
1540  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1541  afwMath.MEDIAN | afwMath.STDEVCLIP)
1542  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1543  qaStats.getValue(afwMath.MEDIAN))
1544  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1545  qaStats.getValue(afwMath.STDEVCLIP))
1546  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1547  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1548  qaStats.getValue(afwMath.STDEVCLIP))
1549  if self.config.maskNegativeVariance:
1550  self.maskNegativeVariancemaskNegativeVariance(ccdExposure)
1551 
1552  if self.doLinearizedoLinearize(ccd):
1553  self.log.info("Applying linearizer.")
1554  linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1555  detector=ccd, log=self.log)
1556 
1557  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1558  self.log.info("Applying crosstalk correction.")
1559  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1560  crosstalkSources=crosstalkSources, isTrimmed=True)
1561  self.debugViewdebugView(ccdExposure, "doCrosstalk")
1562 
1563  # Masking block. Optionally mask known defects, NAN/inf pixels,
1564  # widen trails, and do anything else the camera needs. Saturated and
1565  # suspect pixels have already been masked.
1566  if self.config.doDefect:
1567  self.log.info("Masking defects.")
1568  self.maskDefectmaskDefect(ccdExposure, defects)
1569 
1570  if self.config.numEdgeSuspect > 0:
1571  self.log.info("Masking edges as SUSPECT.")
1572  self.maskEdgesmaskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1573  maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
1574 
1575  if self.config.doNanMasking:
1576  self.log.info("Masking non-finite (NAN, inf) value pixels.")
1577  self.maskNanmaskNan(ccdExposure)
1578 
1579  if self.config.doWidenSaturationTrails:
1580  self.log.info("Widening saturation trails.")
1581  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1582 
1583  if self.config.doCameraSpecificMasking:
1584  self.log.info("Masking regions for camera specific reasons.")
1585  self.masking.run(ccdExposure)
1586 
1587  if self.config.doBrighterFatter:
1588  # We need to apply flats and darks before we can interpolate, and
1589  # we need to interpolate before we do B-F, but we do B-F without
1590  # the flats and darks applied so we can work in units of electrons
1591  # or holes. This context manager applies and then removes the darks
1592  # and flats.
1593  #
1594  # We also do not want to interpolate values here, so operate on
1595  # temporary images so we can apply only the BF-correction and roll
1596  # back the interpolation.
1597  interpExp = ccdExposure.clone()
1598  with self.flatContextflatContext(interpExp, flat, dark):
1599  isrFunctions.interpolateFromMask(
1600  maskedImage=interpExp.getMaskedImage(),
1601  fwhm=self.config.fwhm,
1602  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1603  maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1604  )
1605  bfExp = interpExp.clone()
1606 
1607  self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.",
1608  type(bfKernel), type(bfGains))
1609  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1610  self.config.brighterFatterMaxIter,
1611  self.config.brighterFatterThreshold,
1612  self.config.brighterFatterApplyGain,
1613  bfGains)
1614  if bfResults[1] == self.config.brighterFatterMaxIter:
1615  self.log.warning("Brighter-fatter correction did not converge, final difference %f.",
1616  bfResults[0])
1617  else:
1618  self.log.info("Finished brighter-fatter correction in %d iterations.",
1619  bfResults[1])
1620  image = ccdExposure.getMaskedImage().getImage()
1621  bfCorr = bfExp.getMaskedImage().getImage()
1622  bfCorr -= interpExp.getMaskedImage().getImage()
1623  image += bfCorr
1624 
1625  # Applying the brighter-fatter correction applies a
1626  # convolution to the science image. At the edges this
1627  # convolution may not have sufficient valid pixels to
1628  # produce a valid correction. Mark pixels within the size
1629  # of the brighter-fatter kernel as EDGE to warn of this
1630  # fact.
1631  self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1632  self.maskEdgesmaskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1633  maskPlane="EDGE")
1634 
1635  if self.config.brighterFatterMaskGrowSize > 0:
1636  self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1637  for maskPlane in self.config.brighterFatterMaskListToInterpolate:
1638  isrFunctions.growMasks(ccdExposure.getMask(),
1639  radius=self.config.brighterFatterMaskGrowSize,
1640  maskNameList=maskPlane,
1641  maskValue=maskPlane)
1642 
1643  self.debugViewdebugView(ccdExposure, "doBrighterFatter")
1644 
1645  if self.config.doDark:
1646  self.log.info("Applying dark correction.")
1647  self.darkCorrectiondarkCorrection(ccdExposure, dark)
1648  self.debugViewdebugView(ccdExposure, "doDark")
1649 
1650  if self.config.doFringe and not self.config.fringeAfterFlat:
1651  self.log.info("Applying fringe correction before flat.")
1652  self.fringe.run(ccdExposure, **fringes.getDict())
1653  self.debugViewdebugView(ccdExposure, "doFringe")
1654 
1655  if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1656  self.log.info("Checking strayLight correction.")
1657  self.strayLight.run(ccdExposure, strayLightData)
1658  self.debugViewdebugView(ccdExposure, "doStrayLight")
1659 
1660  if self.config.doFlat:
1661  self.log.info("Applying flat correction.")
1662  self.flatCorrectionflatCorrection(ccdExposure, flat)
1663  self.debugViewdebugView(ccdExposure, "doFlat")
1664 
1665  if self.config.doApplyGains:
1666  self.log.info("Applying gain correction instead of flat.")
1667  if self.config.usePtcGains:
1668  self.log.info("Using gains from the Photon Transfer Curve.")
1669  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1670  ptcGains=ptc.gain)
1671  else:
1672  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1673 
1674  if self.config.doFringe and self.config.fringeAfterFlat:
1675  self.log.info("Applying fringe correction after flat.")
1676  self.fringe.run(ccdExposure, **fringes.getDict())
1677 
1678  if self.config.doVignette:
1679  self.log.info("Constructing Vignette polygon.")
1680  self.vignettePolygonvignettePolygon = self.vignette.run(ccdExposure)
1681 
1682  if self.config.vignette.doWriteVignettePolygon:
1683  self.setValidPolygonIntersectsetValidPolygonIntersect(ccdExposure, self.vignettePolygonvignettePolygon)
1684 
1685  if self.config.doAttachTransmissionCurve:
1686  self.log.info("Adding transmission curves.")
1687  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1688  filterTransmission=filterTransmission,
1689  sensorTransmission=sensorTransmission,
1690  atmosphereTransmission=atmosphereTransmission)
1691 
1692  flattenedThumb = None
1693  if self.config.qa.doThumbnailFlattened:
1694  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1695 
1696  if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters:
1697  self.log.info("Performing illumination correction.")
1698  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1699  illumMaskedImage, illumScale=self.config.illumScale,
1700  trimToFit=self.config.doTrimToMatchCalib)
1701 
1702  preInterpExp = None
1703  if self.config.doSaveInterpPixels:
1704  preInterpExp = ccdExposure.clone()
1705 
1706  # Reset and interpolate bad pixels.
1707  #
1708  # Large contiguous bad regions (which should have the BAD mask
1709  # bit set) should have their values set to the image median.
1710  # This group should include defects and bad amplifiers. As the
1711  # area covered by these defects are large, there's little
1712  # reason to expect that interpolation would provide a more
1713  # useful value.
1714  #
1715  # Smaller defects can be safely interpolated after the larger
1716  # regions have had their pixel values reset. This ensures
1717  # that the remaining defects adjacent to bad amplifiers (as an
1718  # example) do not attempt to interpolate extreme values.
1719  if self.config.doSetBadRegions:
1720  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1721  if badPixelCount > 0:
1722  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1723 
1724  if self.config.doInterpolate:
1725  self.log.info("Interpolating masked pixels.")
1726  isrFunctions.interpolateFromMask(
1727  maskedImage=ccdExposure.getMaskedImage(),
1728  fwhm=self.config.fwhm,
1729  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1730  maskNameList=list(self.config.maskListToInterpolate)
1731  )
1732 
1733  self.roughZeroPointroughZeroPoint(ccdExposure)
1734 
1735  # correct for amp offsets within the CCD
1736  if self.config.doAmpOffset:
1737  self.log.info("Correcting amp offsets.")
1738  self.ampOffset.run(ccdExposure)
1739 
1740  if self.config.doMeasureBackground:
1741  self.log.info("Measuring background level.")
1742  self.measureBackgroundmeasureBackground(ccdExposure, self.config.qa)
1743 
1744  if self.config.qa is not None and self.config.qa.saveStats is True:
1745  for amp in ccd:
1746  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1747  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1748  afwMath.MEDIAN | afwMath.STDEVCLIP)
1749  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1750  qaStats.getValue(afwMath.MEDIAN))
1751  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1752  qaStats.getValue(afwMath.STDEVCLIP))
1753  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1754  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1755  qaStats.getValue(afwMath.STDEVCLIP))
1756 
1757  self.debugViewdebugView(ccdExposure, "postISRCCD")
1758 
1759  return pipeBase.Struct(
1760  exposure=ccdExposure,
1761  ossThumb=ossThumb,
1762  flattenedThumb=flattenedThumb,
1763 
1764  preInterpExposure=preInterpExp,
1765  outputExposure=ccdExposure,
1766  outputOssThumbnail=ossThumb,
1767  outputFlattenedThumbnail=flattenedThumb,
1768  )
1769 
1770  @timeMethod
1771  def runDataRef(self, sensorRef):
1772  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1773 
1774  This method contains the `CmdLineTask` interface to the ISR
1775  processing. All IO is handled here, freeing the `run()` method
1776  to manage only pixel-level calculations. The steps performed
1777  are:
1778  - Read in necessary detrending/isr/calibration data.
1779  - Process raw exposure in `run()`.
1780  - Persist the ISR-corrected exposure as "postISRCCD" if
1781  config.doWrite=True.
1782 
1783  Parameters
1784  ----------
1785  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1786  DataRef of the detector data to be processed
1787 
1788  Returns
1789  -------
1790  result : `lsst.pipe.base.Struct`
1791  Result struct with component:
1792  - ``exposure`` : `afw.image.Exposure`
1793  The fully ISR corrected exposure.
1794 
1795  Raises
1796  ------
1797  RuntimeError
1798  Raised if a configuration option is set to True, but the
1799  required calibration data does not exist.
1800 
1801  """
1802  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1803 
1804  ccdExposure = sensorRef.get(self.config.datasetType)
1805 
1806  camera = sensorRef.get("camera")
1807  isrData = self.readIsrDatareadIsrData(sensorRef, ccdExposure)
1808 
1809  result = self.runrun(ccdExposure, camera=camera, **isrData.getDict())
1810 
1811  if self.config.doWrite:
1812  sensorRef.put(result.exposure, "postISRCCD")
1813  if result.preInterpExposure is not None:
1814  sensorRef.put(result.preInterpExposure, "postISRCCD_uninterpolated")
1815  if result.ossThumb is not None:
1816  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1817  if result.flattenedThumb is not None:
1818  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1819 
1820  return result
1821 
1822  def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1823  """Retrieve a calibration dataset for removing instrument signature.
1824 
1825  Parameters
1826  ----------
1827 
1828  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1829  DataRef of the detector data to find calibration datasets
1830  for.
1831  datasetType : `str`
1832  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1833  dateObs : `str`, optional
1834  Date of the observation. Used to correct butler failures
1835  when using fallback filters.
1836  immediate : `Bool`
1837  If True, disable butler proxies to enable error handling
1838  within this routine.
1839 
1840  Returns
1841  -------
1842  exposure : `lsst.afw.image.Exposure`
1843  Requested calibration frame.
1844 
1845  Raises
1846  ------
1847  RuntimeError
1848  Raised if no matching calibration frame can be found.
1849  """
1850  try:
1851  exp = dataRef.get(datasetType, immediate=immediate)
1852  except Exception as exc1:
1853  if not self.config.fallbackFilterName:
1854  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1855  try:
1856  if self.config.useFallbackDate and dateObs:
1857  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1858  dateObs=dateObs, immediate=immediate)
1859  else:
1860  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1861  except Exception as exc2:
1862  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1863  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1864  self.log.warning("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1865 
1866  if self.config.doAssembleIsrExposures:
1867  exp = self.assembleCcd.assembleCcd(exp)
1868  return exp
1869 
1870  def ensureExposure(self, inputExp, camera=None, detectorNum=None):
1871  """Ensure that the data returned by Butler is a fully constructed exp.
1872 
1873  ISR requires exposure-level image data for historical reasons, so if we
1874  did not recieve that from Butler, construct it from what we have,
1875  modifying the input in place.
1876 
1877  Parameters
1878  ----------
1879  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`,
1880  or `lsst.afw.image.ImageF`
1881  The input data structure obtained from Butler.
1882  camera : `lsst.afw.cameraGeom.camera`, optional
1883  The camera associated with the image. Used to find the appropriate
1884  detector if detector is not already set.
1885  detectorNum : `int`, optional
1886  The detector in the camera to attach, if the detector is not
1887  already set.
1888 
1889  Returns
1890  -------
1891  inputExp : `lsst.afw.image.Exposure`
1892  The re-constructed exposure, with appropriate detector parameters.
1893 
1894  Raises
1895  ------
1896  TypeError
1897  Raised if the input data cannot be used to construct an exposure.
1898  """
1899  if isinstance(inputExp, afwImage.DecoratedImageU):
1900  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1901  elif isinstance(inputExp, afwImage.ImageF):
1902  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1903  elif isinstance(inputExp, afwImage.MaskedImageF):
1904  inputExp = afwImage.makeExposure(inputExp)
1905  elif isinstance(inputExp, afwImage.Exposure):
1906  pass
1907  elif inputExp is None:
1908  # Assume this will be caught by the setup if it is a problem.
1909  return inputExp
1910  else:
1911  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1912  (type(inputExp), ))
1913 
1914  if inputExp.getDetector() is None:
1915  if camera is None or detectorNum is None:
1916  raise RuntimeError('Must supply both a camera and detector number when using exposures '
1917  'without a detector set.')
1918  inputExp.setDetector(camera[detectorNum])
1919 
1920  return inputExp
1921 
1922  def convertIntToFloat(self, exposure):
1923  """Convert exposure image from uint16 to float.
1924 
1925  If the exposure does not need to be converted, the input is
1926  immediately returned. For exposures that are converted to use
1927  floating point pixels, the variance is set to unity and the
1928  mask to zero.
1929 
1930  Parameters
1931  ----------
1932  exposure : `lsst.afw.image.Exposure`
1933  The raw exposure to be converted.
1934 
1935  Returns
1936  -------
1937  newexposure : `lsst.afw.image.Exposure`
1938  The input ``exposure``, converted to floating point pixels.
1939 
1940  Raises
1941  ------
1942  RuntimeError
1943  Raised if the exposure type cannot be converted to float.
1944 
1945  """
1946  if isinstance(exposure, afwImage.ExposureF):
1947  # Nothing to be done
1948  self.log.debug("Exposure already of type float.")
1949  return exposure
1950  if not hasattr(exposure, "convertF"):
1951  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1952 
1953  newexposure = exposure.convertF()
1954  newexposure.variance[:] = 1
1955  newexposure.mask[:] = 0x0
1956 
1957  return newexposure
1958 
1959  def maskAmplifier(self, ccdExposure, amp, defects):
1960  """Identify bad amplifiers, saturated and suspect pixels.
1961 
1962  Parameters
1963  ----------
1964  ccdExposure : `lsst.afw.image.Exposure`
1965  Input exposure to be masked.
1966  amp : `lsst.afw.table.AmpInfoCatalog`
1967  Catalog of parameters defining the amplifier on this
1968  exposure to mask.
1969  defects : `lsst.ip.isr.Defects`
1970  List of defects. Used to determine if the entire
1971  amplifier is bad.
1972 
1973  Returns
1974  -------
1975  badAmp : `Bool`
1976  If this is true, the entire amplifier area is covered by
1977  defects and unusable.
1978 
1979  """
1980  maskedImage = ccdExposure.getMaskedImage()
1981 
1982  badAmp = False
1983 
1984  # Check if entire amp region is defined as a defect
1985  # NB: need to use amp.getBBox() for correct comparison with current
1986  # defects definition.
1987  if defects is not None:
1988  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1989 
1990  # In the case of a bad amp, we will set mask to "BAD"
1991  # (here use amp.getRawBBox() for correct association with pixels in
1992  # current ccdExposure).
1993  if badAmp:
1994  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1995  afwImage.PARENT)
1996  maskView = dataView.getMask()
1997  maskView |= maskView.getPlaneBitMask("BAD")
1998  del maskView
1999  return badAmp
2000 
2001  # Mask remaining defects after assembleCcd() to allow for defects that
2002  # cross amplifier boundaries. Saturation and suspect pixels can be
2003  # masked now, though.
2004  limits = dict()
2005  if self.config.doSaturation and not badAmp:
2006  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
2007  if self.config.doSuspect and not badAmp:
2008  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
2009  if math.isfinite(self.config.saturation):
2010  limits.update({self.config.saturatedMaskName: self.config.saturation})
2011 
2012  for maskName, maskThreshold in limits.items():
2013  if not math.isnan(maskThreshold):
2014  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2015  isrFunctions.makeThresholdMask(
2016  maskedImage=dataView,
2017  threshold=maskThreshold,
2018  growFootprints=0,
2019  maskName=maskName
2020  )
2021 
2022  # Determine if we've fully masked this amplifier with SUSPECT and
2023  # SAT pixels.
2024  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
2025  afwImage.PARENT)
2026  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
2027  self.config.suspectMaskName])
2028  if numpy.all(maskView.getArray() & maskVal > 0):
2029  badAmp = True
2030  maskView |= maskView.getPlaneBitMask("BAD")
2031 
2032  return badAmp
2033 
2034  def overscanCorrection(self, ccdExposure, amp):
2035  """Apply overscan correction in place.
2036 
2037  This method does initial pixel rejection of the overscan
2038  region. The overscan can also be optionally segmented to
2039  allow for discontinuous overscan responses to be fit
2040  separately. The actual overscan subtraction is performed by
2041  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
2042  which is called here after the amplifier is preprocessed.
2043 
2044  Parameters
2045  ----------
2046  ccdExposure : `lsst.afw.image.Exposure`
2047  Exposure to have overscan correction performed.
2048  amp : `lsst.afw.cameraGeom.Amplifer`
2049  The amplifier to consider while correcting the overscan.
2050 
2051  Returns
2052  -------
2053  overscanResults : `lsst.pipe.base.Struct`
2054  Result struct with components:
2055  - ``imageFit`` : scalar or `lsst.afw.image.Image`
2056  Value or fit subtracted from the amplifier image data.
2057  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
2058  Value or fit subtracted from the overscan image data.
2059  - ``overscanImage`` : `lsst.afw.image.Image`
2060  Image of the overscan region with the overscan
2061  correction applied. This quantity is used to estimate
2062  the amplifier read noise empirically.
2063 
2064  Raises
2065  ------
2066  RuntimeError
2067  Raised if the ``amp`` does not contain raw pixel information.
2068 
2069  See Also
2070  --------
2071  lsst.ip.isr.isrFunctions.overscanCorrection
2072  """
2073  if amp.getRawHorizontalOverscanBBox().isEmpty():
2074  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
2075  return None
2076 
2077  statControl = afwMath.StatisticsControl()
2078  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2079 
2080  # Determine the bounding boxes
2081  dataBBox = amp.getRawDataBBox()
2082  oscanBBox = amp.getRawHorizontalOverscanBBox()
2083  dx0 = 0
2084  dx1 = 0
2085 
2086  prescanBBox = amp.getRawPrescanBBox()
2087  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
2088  dx0 += self.config.overscanNumLeadingColumnsToSkip
2089  dx1 -= self.config.overscanNumTrailingColumnsToSkip
2090  else:
2091  dx0 += self.config.overscanNumTrailingColumnsToSkip
2092  dx1 -= self.config.overscanNumLeadingColumnsToSkip
2093 
2094  # Determine if we need to work on subregions of the amplifier
2095  # and overscan.
2096  imageBBoxes = []
2097  overscanBBoxes = []
2098 
2099  if ((self.config.overscanBiasJump
2100  and self.config.overscanBiasJumpLocation)
2101  and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2102  and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
2103  self.config.overscanBiasJumpDevices)):
2104  if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
2105  yLower = self.config.overscanBiasJumpLocation
2106  yUpper = dataBBox.getHeight() - yLower
2107  else:
2108  yUpper = self.config.overscanBiasJumpLocation
2109  yLower = dataBBox.getHeight() - yUpper
2110 
2111  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2112  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
2113  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2114  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2115  yLower)))
2116 
2117  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
2118  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
2119  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
2120  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2121  yUpper)))
2122  else:
2123  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2124  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
2125  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2126  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2127  oscanBBox.getHeight())))
2128 
2129  # Perform overscan correction on subregions, ensuring saturated
2130  # pixels are masked.
2131  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
2132  ampImage = ccdExposure.maskedImage[imageBBox]
2133  overscanImage = ccdExposure.maskedImage[overscanBBox]
2134 
2135  overscanArray = overscanImage.image.array
2136  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2137  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2138  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
2139 
2140  statControl = afwMath.StatisticsControl()
2141  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2142 
2143  overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp)
2144 
2145  # Measure average overscan levels and record them in the metadata.
2146  levelStat = afwMath.MEDIAN
2147  sigmaStat = afwMath.STDEVCLIP
2148 
2149  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2150  self.config.qa.flatness.nIter)
2151  metadata = ccdExposure.getMetadata()
2152  ampNum = amp.getName()
2153  # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
2154  if isinstance(overscanResults.overscanFit, float):
2155  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2156  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2157  else:
2158  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2159  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2160  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2161 
2162  return overscanResults
2163 
2164  def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None):
2165  """Set the variance plane using the gain and read noise
2166 
2167  The read noise is calculated from the ``overscanImage`` if the
2168  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2169  the value from the amplifier data is used.
2170 
2171  Parameters
2172  ----------
2173  ampExposure : `lsst.afw.image.Exposure`
2174  Exposure to process.
2175  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2176  Amplifier detector data.
2177  overscanImage : `lsst.afw.image.MaskedImage`, optional.
2178  Image of overscan, required only for empirical read noise.
2179  ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
2180  PTC dataset containing the gains and read noise.
2181 
2182 
2183  Raises
2184  ------
2185  RuntimeError
2186  Raised if either ``usePtcGains`` of ``usePtcReadNoise``
2187  are ``True``, but ptcDataset is not provided.
2188 
2189  Raised if ```doEmpiricalReadNoise`` is ``True`` but
2190  ``overscanImage`` is ``None``.
2191 
2192  See also
2193  --------
2194  lsst.ip.isr.isrFunctions.updateVariance
2195  """
2196  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2197  if self.config.usePtcGains:
2198  if ptcDataset is None:
2199  raise RuntimeError("No ptcDataset provided to use PTC gains.")
2200  else:
2201  gain = ptcDataset.gain[amp.getName()]
2202  self.log.info("Using gain from Photon Transfer Curve.")
2203  else:
2204  gain = amp.getGain()
2205 
2206  if math.isnan(gain):
2207  gain = 1.0
2208  self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2209  elif gain <= 0:
2210  patchedGain = 1.0
2211  self.log.warning("Gain for amp %s == %g <= 0; setting to %f.",
2212  amp.getName(), gain, patchedGain)
2213  gain = patchedGain
2214 
2215  if self.config.doEmpiricalReadNoise and overscanImage is None:
2216  raise RuntimeError("Overscan is none for EmpiricalReadNoise.")
2217 
2218  if self.config.doEmpiricalReadNoise and overscanImage is not None:
2219  stats = afwMath.StatisticsControl()
2220  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2221  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2222  self.log.info("Calculated empirical read noise for amp %s: %f.",
2223  amp.getName(), readNoise)
2224  elif self.config.usePtcReadNoise:
2225  if ptcDataset is None:
2226  raise RuntimeError("No ptcDataset provided to use PTC readnoise.")
2227  else:
2228  readNoise = ptcDataset.noise[amp.getName()]
2229  self.log.info("Using read noise from Photon Transfer Curve.")
2230  else:
2231  readNoise = amp.getReadNoise()
2232 
2233  isrFunctions.updateVariance(
2234  maskedImage=ampExposure.getMaskedImage(),
2235  gain=gain,
2236  readNoise=readNoise,
2237  )
2238 
2239  def maskNegativeVariance(self, exposure):
2240  """Identify and mask pixels with negative variance values.
2241 
2242  Parameters
2243  ----------
2244  exposure : `lsst.afw.image.Exposure`
2245  Exposure to process.
2246 
2247  See Also
2248  --------
2249  lsst.ip.isr.isrFunctions.updateVariance
2250  """
2251  maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
2252  bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
2253  exposure.mask.array[bad] |= maskPlane
2254 
2255  def darkCorrection(self, exposure, darkExposure, invert=False):
2256  """Apply dark correction in place.
2257 
2258  Parameters
2259  ----------
2260  exposure : `lsst.afw.image.Exposure`
2261  Exposure to process.
2262  darkExposure : `lsst.afw.image.Exposure`
2263  Dark exposure of the same size as ``exposure``.
2264  invert : `Bool`, optional
2265  If True, re-add the dark to an already corrected image.
2266 
2267  Raises
2268  ------
2269  RuntimeError
2270  Raised if either ``exposure`` or ``darkExposure`` do not
2271  have their dark time defined.
2272 
2273  See Also
2274  --------
2275  lsst.ip.isr.isrFunctions.darkCorrection
2276  """
2277  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2278  if math.isnan(expScale):
2279  raise RuntimeError("Exposure darktime is NAN.")
2280  if darkExposure.getInfo().getVisitInfo() is not None \
2281  and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2282  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2283  else:
2284  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
2285  # so getDarkTime() does not exist.
2286  self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2287  darkScale = 1.0
2288 
2289  isrFunctions.darkCorrection(
2290  maskedImage=exposure.getMaskedImage(),
2291  darkMaskedImage=darkExposure.getMaskedImage(),
2292  expScale=expScale,
2293  darkScale=darkScale,
2294  invert=invert,
2295  trimToFit=self.config.doTrimToMatchCalib
2296  )
2297 
2298  def doLinearize(self, detector):
2299  """Check if linearization is needed for the detector cameraGeom.
2300 
2301  Checks config.doLinearize and the linearity type of the first
2302  amplifier.
2303 
2304  Parameters
2305  ----------
2306  detector : `lsst.afw.cameraGeom.Detector`
2307  Detector to get linearity type from.
2308 
2309  Returns
2310  -------
2311  doLinearize : `Bool`
2312  If True, linearization should be performed.
2313  """
2314  return self.config.doLinearize and \
2315  detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2316 
2317  def flatCorrection(self, exposure, flatExposure, invert=False):
2318  """Apply flat correction in place.
2319 
2320  Parameters
2321  ----------
2322  exposure : `lsst.afw.image.Exposure`
2323  Exposure to process.
2324  flatExposure : `lsst.afw.image.Exposure`
2325  Flat exposure of the same size as ``exposure``.
2326  invert : `Bool`, optional
2327  If True, unflatten an already flattened image.
2328 
2329  See Also
2330  --------
2331  lsst.ip.isr.isrFunctions.flatCorrection
2332  """
2333  isrFunctions.flatCorrection(
2334  maskedImage=exposure.getMaskedImage(),
2335  flatMaskedImage=flatExposure.getMaskedImage(),
2336  scalingType=self.config.flatScalingType,
2337  userScale=self.config.flatUserScale,
2338  invert=invert,
2339  trimToFit=self.config.doTrimToMatchCalib
2340  )
2341 
2342  def saturationDetection(self, exposure, amp):
2343  """Detect and mask saturated pixels in config.saturatedMaskName.
2344 
2345  Parameters
2346  ----------
2347  exposure : `lsst.afw.image.Exposure`
2348  Exposure to process. Only the amplifier DataSec is processed.
2349  amp : `lsst.afw.table.AmpInfoCatalog`
2350  Amplifier detector data.
2351 
2352  See Also
2353  --------
2354  lsst.ip.isr.isrFunctions.makeThresholdMask
2355  """
2356  if not math.isnan(amp.getSaturation()):
2357  maskedImage = exposure.getMaskedImage()
2358  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2359  isrFunctions.makeThresholdMask(
2360  maskedImage=dataView,
2361  threshold=amp.getSaturation(),
2362  growFootprints=0,
2363  maskName=self.config.saturatedMaskName,
2364  )
2365 
2366  def saturationInterpolation(self, exposure):
2367  """Interpolate over saturated pixels, in place.
2368 
2369  This method should be called after `saturationDetection`, to
2370  ensure that the saturated pixels have been identified in the
2371  SAT mask. It should also be called after `assembleCcd`, since
2372  saturated regions may cross amplifier boundaries.
2373 
2374  Parameters
2375  ----------
2376  exposure : `lsst.afw.image.Exposure`
2377  Exposure to process.
2378 
2379  See Also
2380  --------
2381  lsst.ip.isr.isrTask.saturationDetection
2382  lsst.ip.isr.isrFunctions.interpolateFromMask
2383  """
2384  isrFunctions.interpolateFromMask(
2385  maskedImage=exposure.getMaskedImage(),
2386  fwhm=self.config.fwhm,
2387  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2388  maskNameList=list(self.config.saturatedMaskName),
2389  )
2390 
2391  def suspectDetection(self, exposure, amp):
2392  """Detect and mask suspect pixels in config.suspectMaskName.
2393 
2394  Parameters
2395  ----------
2396  exposure : `lsst.afw.image.Exposure`
2397  Exposure to process. Only the amplifier DataSec is processed.
2398  amp : `lsst.afw.table.AmpInfoCatalog`
2399  Amplifier detector data.
2400 
2401  See Also
2402  --------
2403  lsst.ip.isr.isrFunctions.makeThresholdMask
2404 
2405  Notes
2406  -----
2407  Suspect pixels are pixels whose value is greater than
2408  amp.getSuspectLevel(). This is intended to indicate pixels that may be
2409  affected by unknown systematics; for example if non-linearity
2410  corrections above a certain level are unstable then that would be a
2411  useful value for suspectLevel. A value of `nan` indicates that no such
2412  level exists and no pixels are to be masked as suspicious.
2413  """
2414  suspectLevel = amp.getSuspectLevel()
2415  if math.isnan(suspectLevel):
2416  return
2417 
2418  maskedImage = exposure.getMaskedImage()
2419  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2420  isrFunctions.makeThresholdMask(
2421  maskedImage=dataView,
2422  threshold=suspectLevel,
2423  growFootprints=0,
2424  maskName=self.config.suspectMaskName,
2425  )
2426 
2427  def maskDefect(self, exposure, defectBaseList):
2428  """Mask defects using mask plane "BAD", in place.
2429 
2430  Parameters
2431  ----------
2432  exposure : `lsst.afw.image.Exposure`
2433  Exposure to process.
2434  defectBaseList : `lsst.ip.isr.Defects` or `list` of
2435  `lsst.afw.image.DefectBase`.
2436  List of defects to mask.
2437 
2438  Notes
2439  -----
2440  Call this after CCD assembly, since defects may cross amplifier
2441  boundaries.
2442  """
2443  maskedImage = exposure.getMaskedImage()
2444  if not isinstance(defectBaseList, Defects):
2445  # Promotes DefectBase to Defect
2446  defectList = Defects(defectBaseList)
2447  else:
2448  defectList = defectBaseList
2449  defectList.maskPixels(maskedImage, maskName="BAD")
2450 
2451  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2452  """Mask edge pixels with applicable mask plane.
2453 
2454  Parameters
2455  ----------
2456  exposure : `lsst.afw.image.Exposure`
2457  Exposure to process.
2458  numEdgePixels : `int`, optional
2459  Number of edge pixels to mask.
2460  maskPlane : `str`, optional
2461  Mask plane name to use.
2462  level : `str`, optional
2463  Level at which to mask edges.
2464  """
2465  maskedImage = exposure.getMaskedImage()
2466  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2467 
2468  if numEdgePixels > 0:
2469  if level == 'DETECTOR':
2470  boxes = [maskedImage.getBBox()]
2471  elif level == 'AMP':
2472  boxes = [amp.getBBox() for amp in exposure.getDetector()]
2473 
2474  for box in boxes:
2475  # This makes a bbox numEdgeSuspect pixels smaller than the
2476  # image on each side
2477  subImage = maskedImage[box]
2478  box.grow(-numEdgePixels)
2479  # Mask pixels outside box
2480  SourceDetectionTask.setEdgeBits(
2481  subImage,
2482  box,
2483  maskBitMask)
2484 
2485  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2486  """Mask and interpolate defects using mask plane "BAD", in place.
2487 
2488  Parameters
2489  ----------
2490  exposure : `lsst.afw.image.Exposure`
2491  Exposure to process.
2492  defectBaseList : `lsst.ip.isr.Defects` or `list` of
2493  `lsst.afw.image.DefectBase`.
2494  List of defects to mask and interpolate.
2495 
2496  See Also
2497  --------
2498  lsst.ip.isr.isrTask.maskDefect
2499  """
2500  self.maskDefectmaskDefect(exposure, defectBaseList)
2501  self.maskEdgesmaskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2502  maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2503  isrFunctions.interpolateFromMask(
2504  maskedImage=exposure.getMaskedImage(),
2505  fwhm=self.config.fwhm,
2506  growSaturatedFootprints=0,
2507  maskNameList=["BAD"],
2508  )
2509 
2510  def maskNan(self, exposure):
2511  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2512 
2513  Parameters
2514  ----------
2515  exposure : `lsst.afw.image.Exposure`
2516  Exposure to process.
2517 
2518  Notes
2519  -----
2520  We mask over all non-finite values (NaN, inf), including those
2521  that are masked with other bits (because those may or may not be
2522  interpolated over later, and we want to remove all NaN/infs).
2523  Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2524  preserve the historical name.
2525  """
2526  maskedImage = exposure.getMaskedImage()
2527 
2528  # Find and mask NaNs
2529  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2530  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2531  numNans = maskNans(maskedImage, maskVal)
2532  self.metadata.set("NUMNANS", numNans)
2533  if numNans > 0:
2534  self.log.warning("There were %d unmasked NaNs.", numNans)
2535 
2536  def maskAndInterpolateNan(self, exposure):
2537  """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2538  in place.
2539 
2540  Parameters
2541  ----------
2542  exposure : `lsst.afw.image.Exposure`
2543  Exposure to process.
2544 
2545  See Also
2546  --------
2547  lsst.ip.isr.isrTask.maskNan
2548  """
2549  self.maskNanmaskNan(exposure)
2550  isrFunctions.interpolateFromMask(
2551  maskedImage=exposure.getMaskedImage(),
2552  fwhm=self.config.fwhm,
2553  growSaturatedFootprints=0,
2554  maskNameList=["UNMASKEDNAN"],
2555  )
2556 
2557  def measureBackground(self, exposure, IsrQaConfig=None):
2558  """Measure the image background in subgrids, for quality control.
2559 
2560  Parameters
2561  ----------
2562  exposure : `lsst.afw.image.Exposure`
2563  Exposure to process.
2564  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2565  Configuration object containing parameters on which background
2566  statistics and subgrids to use.
2567  """
2568  if IsrQaConfig is not None:
2569  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2570  IsrQaConfig.flatness.nIter)
2571  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2572  statsControl.setAndMask(maskVal)
2573  maskedImage = exposure.getMaskedImage()
2574  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2575  skyLevel = stats.getValue(afwMath.MEDIAN)
2576  skySigma = stats.getValue(afwMath.STDEVCLIP)
2577  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2578  metadata = exposure.getMetadata()
2579  metadata.set('SKYLEVEL', skyLevel)
2580  metadata.set('SKYSIGMA', skySigma)
2581 
2582  # calcluating flatlevel over the subgrids
2583  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2584  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2585  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2586  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2587  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2588  skyLevels = numpy.zeros((nX, nY))
2589 
2590  for j in range(nY):
2591  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2592  for i in range(nX):
2593  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2594 
2595  xLLC = xc - meshXHalf
2596  yLLC = yc - meshYHalf
2597  xURC = xc + meshXHalf - 1
2598  yURC = yc + meshYHalf - 1
2599 
2600  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2601  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2602 
2603  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2604 
2605  good = numpy.where(numpy.isfinite(skyLevels))
2606  skyMedian = numpy.median(skyLevels[good])
2607  flatness = (skyLevels[good] - skyMedian) / skyMedian
2608  flatness_rms = numpy.std(flatness)
2609  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2610 
2611  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2612  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2613  nX, nY, flatness_pp, flatness_rms)
2614 
2615  metadata.set('FLATNESS_PP', float(flatness_pp))
2616  metadata.set('FLATNESS_RMS', float(flatness_rms))
2617  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2618  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2619  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2620 
2621  def roughZeroPoint(self, exposure):
2622  """Set an approximate magnitude zero point for the exposure.
2623 
2624  Parameters
2625  ----------
2626  exposure : `lsst.afw.image.Exposure`
2627  Exposure to process.
2628  """
2629  filterLabel = exposure.getFilterLabel()
2630  physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2631 
2632  if physicalFilter in self.config.fluxMag0T1:
2633  fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2634  else:
2635  self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter)
2636  fluxMag0 = self.config.defaultFluxMag0T1
2637 
2638  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2639  if not expTime > 0: # handle NaN as well as <= 0
2640  self.log.warning("Non-positive exposure time; skipping rough zero point.")
2641  return
2642 
2643  self.log.info("Setting rough magnitude zero point for filter %s: %f",
2644  physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2645  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2646 
2647  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2648  """Set valid polygon as the intersection of fpPolygon and chip corners.
2649 
2650  Parameters
2651  ----------
2652  ccdExposure : `lsst.afw.image.Exposure`
2653  Exposure to process.
2654  fpPolygon : `lsst.afw.geom.Polygon`
2655  Polygon in focal plane coordinates.
2656  """
2657  # Get ccd corners in focal plane coordinates
2658  ccd = ccdExposure.getDetector()
2659  fpCorners = ccd.getCorners(FOCAL_PLANE)
2660  ccdPolygon = Polygon(fpCorners)
2661 
2662  # Get intersection of ccd corners with fpPolygon
2663  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2664 
2665  # Transform back to pixel positions and build new polygon
2666  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2667  validPolygon = Polygon(ccdPoints)
2668  ccdExposure.getInfo().setValidPolygon(validPolygon)
2669 
2670  @contextmanager
2671  def flatContext(self, exp, flat, dark=None):
2672  """Context manager that applies and removes flats and darks,
2673  if the task is configured to apply them.
2674 
2675  Parameters
2676  ----------
2677  exp : `lsst.afw.image.Exposure`
2678  Exposure to process.
2679  flat : `lsst.afw.image.Exposure`
2680  Flat exposure the same size as ``exp``.
2681  dark : `lsst.afw.image.Exposure`, optional
2682  Dark exposure the same size as ``exp``.
2683 
2684  Yields
2685  ------
2686  exp : `lsst.afw.image.Exposure`
2687  The flat and dark corrected exposure.
2688  """
2689  if self.config.doDark and dark is not None:
2690  self.darkCorrectiondarkCorrection(exp, dark)
2691  if self.config.doFlat:
2692  self.flatCorrectionflatCorrection(exp, flat)
2693  try:
2694  yield exp
2695  finally:
2696  if self.config.doFlat:
2697  self.flatCorrectionflatCorrection(exp, flat, invert=True)
2698  if self.config.doDark and dark is not None:
2699  self.darkCorrectiondarkCorrection(exp, dark, invert=True)
2700 
2701  def debugView(self, exposure, stepname):
2702  """Utility function to examine ISR exposure at different stages.
2703 
2704  Parameters
2705  ----------
2706  exposure : `lsst.afw.image.Exposure`
2707  Exposure to view.
2708  stepname : `str`
2709  State of processing to view.
2710  """
2711  frame = getDebugFrame(self._display, stepname)
2712  if frame:
2713  display = getDisplay(frame)
2714  display.scale('asinh', 'zscale')
2715  display.mtv(exposure)
2716  prompt = "Press Enter to continue [c]... "
2717  while True:
2718  ans = input(prompt).lower()
2719  if ans in ("", "c",):
2720  break
2721 
2722 
2723 class FakeAmp(object):
2724  """A Detector-like object that supports returning gain and saturation level
2725 
2726  This is used when the input exposure does not have a detector.
2727 
2728  Parameters
2729  ----------
2730  exposure : `lsst.afw.image.Exposure`
2731  Exposure to generate a fake amplifier for.
2732  config : `lsst.ip.isr.isrTaskConfig`
2733  Configuration to apply to the fake amplifier.
2734  """
2735 
2736  def __init__(self, exposure, config):
2737  self._bbox_bbox = exposure.getBBox(afwImage.LOCAL)
2738  self._RawHorizontalOverscanBBox_RawHorizontalOverscanBBox = lsst.geom.Box2I()
2739  self._gain_gain = config.gain
2740  self._readNoise_readNoise = config.readNoise
2741  self._saturation_saturation = config.saturation
2742 
2743  def getBBox(self):
2744  return self._bbox_bbox
2745 
2746  def getRawBBox(self):
2747  return self._bbox_bbox
2748 
2750  return self._RawHorizontalOverscanBBox_RawHorizontalOverscanBBox
2751 
2752  def getGain(self):
2753  return self._gain_gain
2754 
2755  def getReadNoise(self):
2756  return self._readNoise_readNoise
2757 
2758  def getSaturation(self):
2759  return self._saturation_saturation
2760 
2761  def getSuspectLevel(self):
2762  return float("NaN")
2763 
2764 
2765 class RunIsrConfig(pexConfig.Config):
2766  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2767 
2768 
2769 class RunIsrTask(pipeBase.CmdLineTask):
2770  """Task to wrap the default IsrTask to allow it to be retargeted.
2771 
2772  The standard IsrTask can be called directly from a command line
2773  program, but doing so removes the ability of the task to be
2774  retargeted. As most cameras override some set of the IsrTask
2775  methods, this would remove those data-specific methods in the
2776  output post-ISR images. This wrapping class fixes the issue,
2777  allowing identical post-ISR images to be generated by both the
2778  processCcd and isrTask code.
2779  """
2780  ConfigClass = RunIsrConfig
2781  _DefaultName = "runIsr"
2782 
2783  def __init__(self, *args, **kwargs):
2784  super().__init__(*args, **kwargs)
2785  self.makeSubtask("isr")
2786 
2787  def runDataRef(self, dataRef):
2788  """
2789  Parameters
2790  ----------
2791  dataRef : `lsst.daf.persistence.ButlerDataRef`
2792  data reference of the detector data to be processed
2793 
2794  Returns
2795  -------
2796  result : `pipeBase.Struct`
2797  Result struct with component:
2798 
2799  - exposure : `lsst.afw.image.Exposure`
2800  Post-ISR processed exposure.
2801  """
2802  return self.isr.runDataRef(dataRef)
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2749
def __init__(self, exposure, config)
Definition: isrTask.py:2736
def __init__(self, *config=None)
Definition: isrTask.py:272
def flatCorrection(self, exposure, flatExposure, invert=False)
Definition: isrTask.py:2317
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2536
def saturationInterpolation(self, exposure)
Definition: isrTask.py:2366
def runDataRef(self, sensorRef)
Definition: isrTask.py:1771
def maskNan(self, exposure)
Definition: isrTask.py:2510
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1959
def debugView(self, exposure, stepname)
Definition: isrTask.py:2701
def ensureExposure(self, inputExp, camera=None, detectorNum=None)
Definition: isrTask.py:1870
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Definition: isrTask.py:1822
def maskNegativeVariance(self, exposure)
Definition: isrTask.py:2239
def saturationDetection(self, exposure, amp)
Definition: isrTask.py:2342
def maskDefect(self, exposure, defectBaseList)
Definition: isrTask.py:2427
def __init__(self, **kwargs)
Definition: isrTask.py:974
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:985
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
Definition: isrTask.py:2451
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:2034
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2557
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2621
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2485
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Definition: isrTask.py:2647
def readIsrData(self, dataRef, rawExposure)
Definition: isrTask.py:1084
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Definition: isrTask.py:1271
def doLinearize(self, detector)
Definition: isrTask.py:2298
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2671
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1922
def suspectDetection(self, exposure, amp)
Definition: isrTask.py:2391
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
Definition: isrTask.py:2164
def darkCorrection(self, exposure, darkExposure, invert=False)
Definition: isrTask.py:2255
def __init__(self, *args, **kwargs)
Definition: isrTask.py:2783
def runDataRef(self, dataRef)
Definition: isrTask.py:2787
def checkFilter(exposure, filterList, log)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
Definition: isrTask.py:64
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35