lsst.ip.isr  20.0.0-2-gbc613e1+1
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36  ReadoutCorner)
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 from lsst.meas.algorithms import Defects
43 
44 from . import isrFunctions
45 from . import isrQa
46 from . import linearize
47 
48 from .assembleCcdTask import AssembleCcdTask
49 from .crosstalk import CrosstalkTask, CrosstalkCalib
50 from .fringe import FringeTask
51 from .isr import maskNans
52 from .masking import MaskingTask
53 from .overscan import OverscanCorrectionTask
54 from .straylight import StrayLightTask
55 from .vignette import VignetteTask
56 from lsst.daf.butler import DataCoordinate, DimensionGraph, DimensionUniverse
57 
58 
59 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
60 
61 
62 def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections):
63  """Lookup function to identify crosstalkSource entries.
64 
65  This should return an empty list under most circumstances. Only
66  when inter-chip crosstalk has been identified should this be
67  populated.
68 
69  This will be unused until DM-25348 resolves the quantum graph
70  generation issue.
71 
72  Parameters
73  ----------
74  datasetType : `str`
75  Dataset to lookup.
76  registry : `lsst.daf.butler.Registry`
77  Butler registry to query.
78  quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79  Data id to transform to identify crosstalkSources. The
80  ``detector`` entry will be stripped.
81  collections : `lsst.daf.butler.CollectionSearch`
82  Collections to search through.
83 
84  Returns
85  -------
86  results : `list` [`lsst.afw.image.Exposure`]
87  List of datasets that match the query that will be used as
88  crosstalkSources.
89  """
90  newDataId = DataCoordinate(DimensionGraph(DimensionUniverse(),
91  names=('instrument', 'exposure')),
92  (quantumDataId['instrument'], quantumDataId['exposure']))
93  results = list(registry.queryDatasets(datasetType,
94  collections=collections,
95  dataId=newDataId,
96  deduplicate=True,
97  expand=True))
98  return(results)
99 
100 
101 class IsrTaskConnections(pipeBase.PipelineTaskConnections,
102  dimensions={"instrument", "exposure", "detector"},
103  defaultTemplates={}):
104  ccdExposure = cT.Input(
105  name="raw",
106  doc="Input exposure to process.",
107  storageClass="Exposure",
108  dimensions=["instrument", "exposure", "detector"],
109  )
110  camera = cT.PrerequisiteInput(
111  name="camera",
112  storageClass="Camera",
113  doc="Input camera to construct complete exposures.",
114  dimensions=["instrument", "calibration_label"],
115  )
116 
117  crosstalk = cT.PrerequisiteInput(
118  name="crosstalk",
119  doc="Input crosstalk object",
120  storageClass="CrosstalkCalib",
121  dimensions=["instrument", "calibration_label", "detector"],
122  )
123  # TODO: DM-25348. This does not work yet to correctly load
124  # possible crosstalk sources.
125  crosstalkSources = cT.PrerequisiteInput(
126  name="isrOverscanCorrected",
127  doc="Overscan corrected input images.",
128  storageClass="Exposure",
129  dimensions=["instrument", "exposure", "detector"],
130  deferLoad=True,
131  multiple=True,
132  lookupFunction=crosstalkSourceLookup,
133  )
134  bias = cT.PrerequisiteInput(
135  name="bias",
136  doc="Input bias calibration.",
137  storageClass="ExposureF",
138  dimensions=["instrument", "calibration_label", "detector"],
139  )
140  dark = cT.PrerequisiteInput(
141  name='dark',
142  doc="Input dark calibration.",
143  storageClass="ExposureF",
144  dimensions=["instrument", "calibration_label", "detector"],
145  )
146  flat = cT.PrerequisiteInput(
147  name="flat",
148  doc="Input flat calibration.",
149  storageClass="ExposureF",
150  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
151  )
152  fringes = cT.PrerequisiteInput(
153  name="fringe",
154  doc="Input fringe calibration.",
155  storageClass="ExposureF",
156  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
157  )
158  strayLightData = cT.PrerequisiteInput(
159  name='yBackground',
160  doc="Input stray light calibration.",
161  storageClass="StrayLightData",
162  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
163  )
164  bfKernel = cT.PrerequisiteInput(
165  name='bfKernel',
166  doc="Input brighter-fatter kernel.",
167  storageClass="NumpyArray",
168  dimensions=["instrument", "calibration_label"],
169  )
170  newBFKernel = cT.PrerequisiteInput(
171  name='brighterFatterKernel',
172  doc="Newer complete kernel + gain solutions.",
173  storageClass="BrighterFatterKernel",
174  dimensions=["instrument", "calibration_label", "detector"],
175  )
176  defects = cT.PrerequisiteInput(
177  name='defects',
178  doc="Input defect tables.",
179  storageClass="Defects",
180  dimensions=["instrument", "calibration_label", "detector"],
181  )
182  opticsTransmission = cT.PrerequisiteInput(
183  name="transmission_optics",
184  storageClass="TransmissionCurve",
185  doc="Transmission curve due to the optics.",
186  dimensions=["instrument", "calibration_label"],
187  )
188  filterTransmission = cT.PrerequisiteInput(
189  name="transmission_filter",
190  storageClass="TransmissionCurve",
191  doc="Transmission curve due to the filter.",
192  dimensions=["instrument", "physical_filter", "calibration_label"],
193  )
194  sensorTransmission = cT.PrerequisiteInput(
195  name="transmission_sensor",
196  storageClass="TransmissionCurve",
197  doc="Transmission curve due to the sensor.",
198  dimensions=["instrument", "calibration_label", "detector"],
199  )
200  atmosphereTransmission = cT.PrerequisiteInput(
201  name="transmission_atmosphere",
202  storageClass="TransmissionCurve",
203  doc="Transmission curve due to the atmosphere.",
204  dimensions=["instrument"],
205  )
206  illumMaskedImage = cT.PrerequisiteInput(
207  name="illum",
208  doc="Input illumination correction.",
209  storageClass="MaskedImageF",
210  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
211  )
212 
213  outputExposure = cT.Output(
214  name='postISRCCD',
215  doc="Output ISR processed exposure.",
216  storageClass="Exposure",
217  dimensions=["instrument", "exposure", "detector"],
218  )
219  preInterpExposure = cT.Output(
220  name='preInterpISRCCD',
221  doc="Output ISR processed exposure, with pixels left uninterpolated.",
222  storageClass="ExposureF",
223  dimensions=["instrument", "exposure", "detector"],
224  )
225  outputOssThumbnail = cT.Output(
226  name="OssThumb",
227  doc="Output Overscan-subtracted thumbnail image.",
228  storageClass="Thumbnail",
229  dimensions=["instrument", "exposure", "detector"],
230  )
231  outputFlattenedThumbnail = cT.Output(
232  name="FlattenedThumb",
233  doc="Output flat-corrected thumbnail image.",
234  storageClass="Thumbnail",
235  dimensions=["instrument", "exposure", "detector"],
236  )
237 
238  def __init__(self, *, config=None):
239  super().__init__(config=config)
240 
241  if config.doBias is not True:
242  self.prerequisiteInputs.discard("bias")
243  if config.doLinearize is not True:
244  self.prerequisiteInputs.discard("linearizer")
245  if config.doCrosstalk is not True:
246  self.inputs.discard("crosstalkSources")
247  self.prerequisiteInputs.discard("crosstalk")
248  if config.doBrighterFatter is not True:
249  self.prerequisiteInputs.discard("bfKernel")
250  self.prerequisiteInputs.discard("newBFKernel")
251  if config.doDefect is not True:
252  self.prerequisiteInputs.discard("defects")
253  if config.doDark is not True:
254  self.prerequisiteInputs.discard("dark")
255  if config.doFlat is not True:
256  self.prerequisiteInputs.discard("flat")
257  if config.doAttachTransmissionCurve is not True:
258  self.prerequisiteInputs.discard("opticsTransmission")
259  self.prerequisiteInputs.discard("filterTransmission")
260  self.prerequisiteInputs.discard("sensorTransmission")
261  self.prerequisiteInputs.discard("atmosphereTransmission")
262  if config.doUseOpticsTransmission is not True:
263  self.prerequisiteInputs.discard("opticsTransmission")
264  if config.doUseFilterTransmission is not True:
265  self.prerequisiteInputs.discard("filterTransmission")
266  if config.doUseSensorTransmission is not True:
267  self.prerequisiteInputs.discard("sensorTransmission")
268  if config.doUseAtmosphereTransmission is not True:
269  self.prerequisiteInputs.discard("atmosphereTransmission")
270  if config.doIlluminationCorrection is not True:
271  self.prerequisiteInputs.discard("illumMaskedImage")
272 
273  if config.doWrite is not True:
274  self.outputs.discard("outputExposure")
275  self.outputs.discard("preInterpExposure")
276  self.outputs.discard("outputFlattenedThumbnail")
277  self.outputs.discard("outputOssThumbnail")
278  if config.doSaveInterpPixels is not True:
279  self.outputs.discard("preInterpExposure")
280  if config.qa.doThumbnailOss is not True:
281  self.outputs.discard("outputOssThumbnail")
282  if config.qa.doThumbnailFlattened is not True:
283  self.outputs.discard("outputFlattenedThumbnail")
284 
285 
286 class IsrTaskConfig(pipeBase.PipelineTaskConfig,
287  pipelineConnections=IsrTaskConnections):
288  """Configuration parameters for IsrTask.
289 
290  Items are grouped in the order in which they are executed by the task.
291  """
292  datasetType = pexConfig.Field(
293  dtype=str,
294  doc="Dataset type for input data; users will typically leave this alone, "
295  "but camera-specific ISR tasks will override it",
296  default="raw",
297  )
298 
299  fallbackFilterName = pexConfig.Field(
300  dtype=str,
301  doc="Fallback default filter name for calibrations.",
302  optional=True
303  )
304  useFallbackDate = pexConfig.Field(
305  dtype=bool,
306  doc="Pass observation date when using fallback filter.",
307  default=False,
308  )
309  expectWcs = pexConfig.Field(
310  dtype=bool,
311  default=True,
312  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
313  )
314  fwhm = pexConfig.Field(
315  dtype=float,
316  doc="FWHM of PSF in arcseconds.",
317  default=1.0,
318  )
319  qa = pexConfig.ConfigField(
320  dtype=isrQa.IsrQaConfig,
321  doc="QA related configuration options.",
322  )
323 
324  # Image conversion configuration
325  doConvertIntToFloat = pexConfig.Field(
326  dtype=bool,
327  doc="Convert integer raw images to floating point values?",
328  default=True,
329  )
330 
331  # Saturated pixel handling.
332  doSaturation = pexConfig.Field(
333  dtype=bool,
334  doc="Mask saturated pixels? NB: this is totally independent of the"
335  " interpolation option - this is ONLY setting the bits in the mask."
336  " To have them interpolated make sure doSaturationInterpolation=True",
337  default=True,
338  )
339  saturatedMaskName = pexConfig.Field(
340  dtype=str,
341  doc="Name of mask plane to use in saturation detection and interpolation",
342  default="SAT",
343  )
344  saturation = pexConfig.Field(
345  dtype=float,
346  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
347  default=float("NaN"),
348  )
349  growSaturationFootprintSize = pexConfig.Field(
350  dtype=int,
351  doc="Number of pixels by which to grow the saturation footprints",
352  default=1,
353  )
354 
355  # Suspect pixel handling.
356  doSuspect = pexConfig.Field(
357  dtype=bool,
358  doc="Mask suspect pixels?",
359  default=False,
360  )
361  suspectMaskName = pexConfig.Field(
362  dtype=str,
363  doc="Name of mask plane to use for suspect pixels",
364  default="SUSPECT",
365  )
366  numEdgeSuspect = pexConfig.Field(
367  dtype=int,
368  doc="Number of edge pixels to be flagged as untrustworthy.",
369  default=0,
370  )
371 
372  # Initial masking options.
373  doSetBadRegions = pexConfig.Field(
374  dtype=bool,
375  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
376  default=True,
377  )
378  badStatistic = pexConfig.ChoiceField(
379  dtype=str,
380  doc="How to estimate the average value for BAD regions.",
381  default='MEANCLIP',
382  allowed={
383  "MEANCLIP": "Correct using the (clipped) mean of good data",
384  "MEDIAN": "Correct using the median of the good data",
385  },
386  )
387 
388  # Overscan subtraction configuration.
389  doOverscan = pexConfig.Field(
390  dtype=bool,
391  doc="Do overscan subtraction?",
392  default=True,
393  )
394  overscan = pexConfig.ConfigurableField(
395  target=OverscanCorrectionTask,
396  doc="Overscan subtraction task for image segments.",
397  )
398 
399  overscanFitType = pexConfig.ChoiceField(
400  dtype=str,
401  doc="The method for fitting the overscan bias level.",
402  default='MEDIAN',
403  allowed={
404  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
405  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
406  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
407  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
408  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
409  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
410  "MEAN": "Correct using the mean of the overscan region",
411  "MEANCLIP": "Correct using a clipped mean of the overscan region",
412  "MEDIAN": "Correct using the median of the overscan region",
413  "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
414  },
415  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
416  " This option will no longer be used, and will be removed after v20.")
417  )
418  overscanOrder = pexConfig.Field(
419  dtype=int,
420  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, "
421  "or number of spline knots if overscan fit type is a spline."),
422  default=1,
423  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
424  " This option will no longer be used, and will be removed after v20.")
425  )
426  overscanNumSigmaClip = pexConfig.Field(
427  dtype=float,
428  doc="Rejection threshold (sigma) for collapsing overscan before fit",
429  default=3.0,
430  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
431  " This option will no longer be used, and will be removed after v20.")
432  )
433  overscanIsInt = pexConfig.Field(
434  dtype=bool,
435  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
436  " and overscan.FitType=MEDIAN_PER_ROW.",
437  default=True,
438  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
439  " This option will no longer be used, and will be removed after v20.")
440  )
441  # These options do not get deprecated, as they define how we slice up the image data.
442  overscanNumLeadingColumnsToSkip = pexConfig.Field(
443  dtype=int,
444  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
445  default=0,
446  )
447  overscanNumTrailingColumnsToSkip = pexConfig.Field(
448  dtype=int,
449  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
450  default=0,
451  )
452  overscanMaxDev = pexConfig.Field(
453  dtype=float,
454  doc="Maximum deviation from the median for overscan",
455  default=1000.0, check=lambda x: x > 0
456  )
457  overscanBiasJump = pexConfig.Field(
458  dtype=bool,
459  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
460  default=False,
461  )
462  overscanBiasJumpKeyword = pexConfig.Field(
463  dtype=str,
464  doc="Header keyword containing information about devices.",
465  default="NO_SUCH_KEY",
466  )
467  overscanBiasJumpDevices = pexConfig.ListField(
468  dtype=str,
469  doc="List of devices that need piecewise overscan correction.",
470  default=(),
471  )
472  overscanBiasJumpLocation = pexConfig.Field(
473  dtype=int,
474  doc="Location of bias jump along y-axis.",
475  default=0,
476  )
477 
478  # Amplifier to CCD assembly configuration
479  doAssembleCcd = pexConfig.Field(
480  dtype=bool,
481  default=True,
482  doc="Assemble amp-level exposures into a ccd-level exposure?"
483  )
484  assembleCcd = pexConfig.ConfigurableField(
485  target=AssembleCcdTask,
486  doc="CCD assembly task",
487  )
488 
489  # General calibration configuration.
490  doAssembleIsrExposures = pexConfig.Field(
491  dtype=bool,
492  default=False,
493  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
494  )
495  doTrimToMatchCalib = pexConfig.Field(
496  dtype=bool,
497  default=False,
498  doc="Trim raw data to match calibration bounding boxes?"
499  )
500 
501  # Bias subtraction.
502  doBias = pexConfig.Field(
503  dtype=bool,
504  doc="Apply bias frame correction?",
505  default=True,
506  )
507  biasDataProductName = pexConfig.Field(
508  dtype=str,
509  doc="Name of the bias data product",
510  default="bias",
511  )
512 
513  # Variance construction
514  doVariance = pexConfig.Field(
515  dtype=bool,
516  doc="Calculate variance?",
517  default=True
518  )
519  gain = pexConfig.Field(
520  dtype=float,
521  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
522  default=float("NaN"),
523  )
524  readNoise = pexConfig.Field(
525  dtype=float,
526  doc="The read noise to use if no Detector is present in the Exposure",
527  default=0.0,
528  )
529  doEmpiricalReadNoise = pexConfig.Field(
530  dtype=bool,
531  default=False,
532  doc="Calculate empirical read noise instead of value from AmpInfo data?"
533  )
534 
535  # Linearization.
536  doLinearize = pexConfig.Field(
537  dtype=bool,
538  doc="Correct for nonlinearity of the detector's response?",
539  default=True,
540  )
541 
542  # Crosstalk.
543  doCrosstalk = pexConfig.Field(
544  dtype=bool,
545  doc="Apply intra-CCD crosstalk correction?",
546  default=False,
547  )
548  doCrosstalkBeforeAssemble = pexConfig.Field(
549  dtype=bool,
550  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
551  default=False,
552  )
553  crosstalk = pexConfig.ConfigurableField(
554  target=CrosstalkTask,
555  doc="Intra-CCD crosstalk correction",
556  )
557 
558  # Masking options.
559  doDefect = pexConfig.Field(
560  dtype=bool,
561  doc="Apply correction for CCD defects, e.g. hot pixels?",
562  default=True,
563  )
564  doNanMasking = pexConfig.Field(
565  dtype=bool,
566  doc="Mask NAN pixels?",
567  default=True,
568  )
569  doWidenSaturationTrails = pexConfig.Field(
570  dtype=bool,
571  doc="Widen bleed trails based on their width?",
572  default=True
573  )
574 
575  # Brighter-Fatter correction.
576  doBrighterFatter = pexConfig.Field(
577  dtype=bool,
578  default=False,
579  doc="Apply the brighter fatter correction"
580  )
581  brighterFatterLevel = pexConfig.ChoiceField(
582  dtype=str,
583  default="DETECTOR",
584  doc="The level at which to correct for brighter-fatter.",
585  allowed={
586  "AMP": "Every amplifier treated separately.",
587  "DETECTOR": "One kernel per detector",
588  }
589  )
590  brighterFatterMaxIter = pexConfig.Field(
591  dtype=int,
592  default=10,
593  doc="Maximum number of iterations for the brighter fatter correction"
594  )
595  brighterFatterThreshold = pexConfig.Field(
596  dtype=float,
597  default=1000,
598  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
599  " absolute value of the difference between the current corrected image and the one"
600  " from the previous iteration summed over all the pixels."
601  )
602  brighterFatterApplyGain = pexConfig.Field(
603  dtype=bool,
604  default=True,
605  doc="Should the gain be applied when applying the brighter fatter correction?"
606  )
607  brighterFatterMaskGrowSize = pexConfig.Field(
608  dtype=int,
609  default=0,
610  doc="Number of pixels to grow the masks listed in config.maskListToInterpolate "
611  " when brighter-fatter correction is applied."
612  )
613 
614  # Dark subtraction.
615  doDark = pexConfig.Field(
616  dtype=bool,
617  doc="Apply dark frame correction?",
618  default=True,
619  )
620  darkDataProductName = pexConfig.Field(
621  dtype=str,
622  doc="Name of the dark data product",
623  default="dark",
624  )
625 
626  # Camera-specific stray light removal.
627  doStrayLight = pexConfig.Field(
628  dtype=bool,
629  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
630  default=False,
631  )
632  strayLight = pexConfig.ConfigurableField(
633  target=StrayLightTask,
634  doc="y-band stray light correction"
635  )
636 
637  # Flat correction.
638  doFlat = pexConfig.Field(
639  dtype=bool,
640  doc="Apply flat field correction?",
641  default=True,
642  )
643  flatDataProductName = pexConfig.Field(
644  dtype=str,
645  doc="Name of the flat data product",
646  default="flat",
647  )
648  flatScalingType = pexConfig.ChoiceField(
649  dtype=str,
650  doc="The method for scaling the flat on the fly.",
651  default='USER',
652  allowed={
653  "USER": "Scale by flatUserScale",
654  "MEAN": "Scale by the inverse of the mean",
655  "MEDIAN": "Scale by the inverse of the median",
656  },
657  )
658  flatUserScale = pexConfig.Field(
659  dtype=float,
660  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
661  default=1.0,
662  )
663  doTweakFlat = pexConfig.Field(
664  dtype=bool,
665  doc="Tweak flats to match observed amplifier ratios?",
666  default=False
667  )
668 
669  # Amplifier normalization based on gains instead of using flats configuration.
670  doApplyGains = pexConfig.Field(
671  dtype=bool,
672  doc="Correct the amplifiers for their gains instead of applying flat correction",
673  default=False,
674  )
675  normalizeGains = pexConfig.Field(
676  dtype=bool,
677  doc="Normalize all the amplifiers in each CCD to have the same median value.",
678  default=False,
679  )
680 
681  # Fringe correction.
682  doFringe = pexConfig.Field(
683  dtype=bool,
684  doc="Apply fringe correction?",
685  default=True,
686  )
687  fringe = pexConfig.ConfigurableField(
688  target=FringeTask,
689  doc="Fringe subtraction task",
690  )
691  fringeAfterFlat = pexConfig.Field(
692  dtype=bool,
693  doc="Do fringe subtraction after flat-fielding?",
694  default=True,
695  )
696 
697  # Initial CCD-level background statistics options.
698  doMeasureBackground = pexConfig.Field(
699  dtype=bool,
700  doc="Measure the background level on the reduced image?",
701  default=False,
702  )
703 
704  # Camera-specific masking configuration.
705  doCameraSpecificMasking = pexConfig.Field(
706  dtype=bool,
707  doc="Mask camera-specific bad regions?",
708  default=False,
709  )
710  masking = pexConfig.ConfigurableField(
711  target=MaskingTask,
712  doc="Masking task."
713  )
714 
715  # Interpolation options.
716 
717  doInterpolate = pexConfig.Field(
718  dtype=bool,
719  doc="Interpolate masked pixels?",
720  default=True,
721  )
722  doSaturationInterpolation = pexConfig.Field(
723  dtype=bool,
724  doc="Perform interpolation over pixels masked as saturated?"
725  " NB: This is independent of doSaturation; if that is False this plane"
726  " will likely be blank, resulting in a no-op here.",
727  default=True,
728  )
729  doNanInterpolation = pexConfig.Field(
730  dtype=bool,
731  doc="Perform interpolation over pixels masked as NaN?"
732  " NB: This is independent of doNanMasking; if that is False this plane"
733  " will likely be blank, resulting in a no-op here.",
734  default=True,
735  )
736  doNanInterpAfterFlat = pexConfig.Field(
737  dtype=bool,
738  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
739  "also have to interpolate them before flat-fielding."),
740  default=False,
741  )
742  maskListToInterpolate = pexConfig.ListField(
743  dtype=str,
744  doc="List of mask planes that should be interpolated.",
745  default=['SAT', 'BAD', 'UNMASKEDNAN'],
746  )
747  doSaveInterpPixels = pexConfig.Field(
748  dtype=bool,
749  doc="Save a copy of the pre-interpolated pixel values?",
750  default=False,
751  )
752 
753  # Default photometric calibration options.
754  fluxMag0T1 = pexConfig.DictField(
755  keytype=str,
756  itemtype=float,
757  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
758  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
759  ))
760  )
761  defaultFluxMag0T1 = pexConfig.Field(
762  dtype=float,
763  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
764  default=pow(10.0, 0.4*28.0)
765  )
766 
767  # Vignette correction configuration.
768  doVignette = pexConfig.Field(
769  dtype=bool,
770  doc="Apply vignetting parameters?",
771  default=False,
772  )
773  vignette = pexConfig.ConfigurableField(
774  target=VignetteTask,
775  doc="Vignetting task.",
776  )
777 
778  # Transmission curve configuration.
779  doAttachTransmissionCurve = pexConfig.Field(
780  dtype=bool,
781  default=False,
782  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
783  )
784  doUseOpticsTransmission = pexConfig.Field(
785  dtype=bool,
786  default=True,
787  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
788  )
789  doUseFilterTransmission = pexConfig.Field(
790  dtype=bool,
791  default=True,
792  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
793  )
794  doUseSensorTransmission = pexConfig.Field(
795  dtype=bool,
796  default=True,
797  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
798  )
799  doUseAtmosphereTransmission = pexConfig.Field(
800  dtype=bool,
801  default=True,
802  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
803  )
804 
805  # Illumination correction.
806  doIlluminationCorrection = pexConfig.Field(
807  dtype=bool,
808  default=False,
809  doc="Perform illumination correction?"
810  )
811  illuminationCorrectionDataProductName = pexConfig.Field(
812  dtype=str,
813  doc="Name of the illumination correction data product.",
814  default="illumcor",
815  )
816  illumScale = pexConfig.Field(
817  dtype=float,
818  doc="Scale factor for the illumination correction.",
819  default=1.0,
820  )
821  illumFilters = pexConfig.ListField(
822  dtype=str,
823  default=[],
824  doc="Only perform illumination correction for these filters."
825  )
826 
827  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
828  doWrite = pexConfig.Field(
829  dtype=bool,
830  doc="Persist postISRCCD?",
831  default=True,
832  )
833 
834  def validate(self):
835  super().validate()
836  if self.doFlat and self.doApplyGains:
837  raise ValueError("You may not specify both doFlat and doApplyGains")
838  if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate:
839  self.config.maskListToInterpolate.append("SAT")
840  if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
841  self.config.maskListToInterpolate.append("UNMASKEDNAN")
842 
843 
844 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
845  """Apply common instrument signature correction algorithms to a raw frame.
846 
847  The process for correcting imaging data is very similar from
848  camera to camera. This task provides a vanilla implementation of
849  doing these corrections, including the ability to turn certain
850  corrections off if they are not needed. The inputs to the primary
851  method, `run()`, are a raw exposure to be corrected and the
852  calibration data products. The raw input is a single chip sized
853  mosaic of all amps including overscans and other non-science
854  pixels. The method `runDataRef()` identifies and defines the
855  calibration data products, and is intended for use by a
856  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
857  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
858  subclassed for different camera, although the most camera specific
859  methods have been split into subtasks that can be redirected
860  appropriately.
861 
862  The __init__ method sets up the subtasks for ISR processing, using
863  the defaults from `lsst.ip.isr`.
864 
865  Parameters
866  ----------
867  args : `list`
868  Positional arguments passed to the Task constructor. None used at this time.
869  kwargs : `dict`, optional
870  Keyword arguments passed on to the Task constructor. None used at this time.
871  """
872  ConfigClass = IsrTaskConfig
873  _DefaultName = "isr"
874 
875  def __init__(self, **kwargs):
876  super().__init__(**kwargs)
877  self.makeSubtask("assembleCcd")
878  self.makeSubtask("crosstalk")
879  self.makeSubtask("strayLight")
880  self.makeSubtask("fringe")
881  self.makeSubtask("masking")
882  self.makeSubtask("overscan")
883  self.makeSubtask("vignette")
884 
885  def runQuantum(self, butlerQC, inputRefs, outputRefs):
886  inputs = butlerQC.get(inputRefs)
887 
888  try:
889  inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
890  except Exception as e:
891  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
892  (inputRefs, e))
893 
894  inputs['isGen3'] = True
895 
896  detector = inputs['ccdExposure'].getDetector()
897 
898  if self.config.doCrosstalk is True:
899  # Crosstalk sources need to be defined by the pipeline
900  # yaml if they exist.
901  if 'crosstalk' in inputs and inputs['crosstalk'] is not None:
902  if not isinstance(inputs['crosstalk'], CrosstalkCalib):
903  inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk'])
904  else:
905  coeffVector = (self.config.crosstalk.crosstalkValues
906  if self.config.crosstalk.useConfigCoefficients else None)
907  crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
908  inputs['crosstalk'] = crosstalkCalib
909  if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0:
910  if 'crosstalkSources' not in inputs:
911  self.log.warn("No crosstalkSources found for chip with interChip terms!")
912 
913  if self.doLinearize(detector) is True:
914  if 'linearizer' in inputs and isinstance(inputs['linearizer'], dict):
915  linearizer = linearize.Linearizer(detector=detector, log=self.log)
916  linearizer.fromYaml(inputs['linearizer'])
917  else:
918  linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), detector=detector,
919  log=self.log)
920  inputs['linearizer'] = linearizer
921 
922  if self.config.doDefect is True:
923  if "defects" in inputs and inputs['defects'] is not None:
924  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
925  # masking expects a list of defects defined by their bounding box
926  if not isinstance(inputs["defects"], Defects):
927  inputs["defects"] = Defects.fromTable(inputs["defects"])
928 
929  # Load the correct style of brighter fatter kernel, and repack
930  # the information as a numpy array.
931  if self.config.doBrighterFatter:
932  brighterFatterKernel = inputs.pop('newBFKernel', None)
933  if brighterFatterKernel is None:
934  brighterFatterKernel = inputs.get('bfKernel', None)
935 
936  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
937  detId = detector.getId()
938  inputs['bfGains'] = brighterFatterKernel.gain
939  # If the kernel is not an ndarray, it's the cp_pipe version
940  # so extract the kernel for this detector, or raise an error
941  if self.config.brighterFatterLevel == 'DETECTOR':
942  if brighterFatterKernel.detectorKernel:
943  inputs['bfKernel'] = brighterFatterKernel.detectorKernel[detId]
944  elif brighterFatterKernel.detectorKernelFromAmpKernels:
945  inputs['bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
946  else:
947  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
948  else:
949  # TODO DM-15631 for implementing this
950  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
951 
952  if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
953  expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
954  inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
955  expId=expId,
956  assembler=self.assembleCcd
957  if self.config.doAssembleIsrExposures else None)
958  else:
959  inputs['fringes'] = pipeBase.Struct(fringes=None)
960 
961  if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
962  if 'strayLightData' not in inputs:
963  inputs['strayLightData'] = None
964 
965  outputs = self.run(**inputs)
966  butlerQC.put(outputs, outputRefs)
967 
968  def readIsrData(self, dataRef, rawExposure):
969  """!Retrieve necessary frames for instrument signature removal.
970 
971  Pre-fetching all required ISR data products limits the IO
972  required by the ISR. Any conflict between the calibration data
973  available and that needed for ISR is also detected prior to
974  doing processing, allowing it to fail quickly.
975 
976  Parameters
977  ----------
978  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
979  Butler reference of the detector data to be processed
980  rawExposure : `afw.image.Exposure`
981  The raw exposure that will later be corrected with the
982  retrieved calibration data; should not be modified in this
983  method.
984 
985  Returns
986  -------
987  result : `lsst.pipe.base.Struct`
988  Result struct with components (which may be `None`):
989  - ``bias``: bias calibration frame (`afw.image.Exposure`)
990  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
991  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
992  - ``dark``: dark calibration frame (`afw.image.Exposure`)
993  - ``flat``: flat calibration frame (`afw.image.Exposure`)
994  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
995  - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
996  - ``fringes``: `lsst.pipe.base.Struct` with components:
997  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
998  - ``seed``: random seed derived from the ccdExposureId for random
999  number generator (`uint32`).
1000  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1001  A ``TransmissionCurve`` that represents the throughput of the optics,
1002  to be evaluated in focal-plane coordinates.
1003  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1004  A ``TransmissionCurve`` that represents the throughput of the filter
1005  itself, to be evaluated in focal-plane coordinates.
1006  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1007  A ``TransmissionCurve`` that represents the throughput of the sensor
1008  itself, to be evaluated in post-assembly trimmed detector coordinates.
1009  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1010  A ``TransmissionCurve`` that represents the throughput of the
1011  atmosphere, assumed to be spatially constant.
1012  - ``strayLightData`` : `object`
1013  An opaque object containing calibration information for
1014  stray-light correction. If `None`, no correction will be
1015  performed.
1016  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1017 
1018  Raises
1019  ------
1020  NotImplementedError :
1021  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1022  """
1023  try:
1024  dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1025  dateObs = dateObs.toPython().isoformat()
1026  except RuntimeError:
1027  self.log.warn("Unable to identify dateObs for rawExposure.")
1028  dateObs = None
1029 
1030  ccd = rawExposure.getDetector()
1031  filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
1032  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
1033  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
1034  if self.config.doBias else None)
1035  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
1036  linearizer = (dataRef.get("linearizer", immediate=True)
1037  if self.doLinearize(ccd) else None)
1038  if linearizer is not None and not isinstance(linearizer, numpy.ndarray):
1039  linearizer.log = self.log
1040  if isinstance(linearizer, numpy.ndarray):
1041  linearizer = linearize.Linearizer(table=linearizer, detector=ccd)
1042 
1043  crosstalkCalib = None
1044  if self.config.doCrosstalk:
1045  try:
1046  crosstalkCalib = dataRef.get("crosstalk", immediate=True)
1047  except NoResults:
1048  coeffVector = (self.config.crosstalk.crosstalkValues
1049  if self.config.crosstalk.useConfigCoefficients else None)
1050  crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1051  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1052  if self.config.doCrosstalk else None)
1053 
1054  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
1055  if self.config.doDark else None)
1056  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName,
1057  dateObs=dateObs)
1058  if self.config.doFlat else None)
1059 
1060  brighterFatterKernel = None
1061  brighterFatterGains = None
1062  if self.config.doBrighterFatter is True:
1063  try:
1064  # Use the new-style cp_pipe version of the kernel if it exists
1065  # If using a new-style kernel, always use the self-consistent
1066  # gains, i.e. the ones inside the kernel object itself
1067  brighterFatterKernel = dataRef.get("brighterFatterKernel")
1068  brighterFatterGains = brighterFatterKernel.gain
1069  self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded")
1070  except NoResults:
1071  try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
1072  brighterFatterKernel = dataRef.get("bfKernel")
1073  self.log.info("Old style bright-fatter kernel (np.array) loaded")
1074  except NoResults:
1075  brighterFatterKernel = None
1076  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1077  # If the kernel is not an ndarray, it's the cp_pipe version
1078  # so extract the kernel for this detector, or raise an error
1079  if self.config.brighterFatterLevel == 'DETECTOR':
1080  if brighterFatterKernel.detectorKernel:
1081  brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
1082  elif brighterFatterKernel.detectorKernelFromAmpKernels:
1083  brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
1084  else:
1085  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1086  else:
1087  # TODO DM-15631 for implementing this
1088  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1089 
1090  defectList = (dataRef.get("defects")
1091  if self.config.doDefect else None)
1092  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
1093  if self.config.doAssembleIsrExposures else None)
1094  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1095  else pipeBase.Struct(fringes=None))
1096 
1097  if self.config.doAttachTransmissionCurve:
1098  opticsTransmission = (dataRef.get("transmission_optics")
1099  if self.config.doUseOpticsTransmission else None)
1100  filterTransmission = (dataRef.get("transmission_filter")
1101  if self.config.doUseFilterTransmission else None)
1102  sensorTransmission = (dataRef.get("transmission_sensor")
1103  if self.config.doUseSensorTransmission else None)
1104  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1105  if self.config.doUseAtmosphereTransmission else None)
1106  else:
1107  opticsTransmission = None
1108  filterTransmission = None
1109  sensorTransmission = None
1110  atmosphereTransmission = None
1111 
1112  if self.config.doStrayLight:
1113  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1114  else:
1115  strayLightData = None
1116 
1117  illumMaskedImage = (self.getIsrExposure(dataRef,
1118  self.config.illuminationCorrectionDataProductName).getMaskedImage()
1119  if (self.config.doIlluminationCorrection
1120  and filterName in self.config.illumFilters)
1121  else None)
1122 
1123  # Struct should include only kwargs to run()
1124  return pipeBase.Struct(bias=biasExposure,
1125  linearizer=linearizer,
1126  crosstalk=crosstalkCalib,
1127  crosstalkSources=crosstalkSources,
1128  dark=darkExposure,
1129  flat=flatExposure,
1130  bfKernel=brighterFatterKernel,
1131  bfGains=brighterFatterGains,
1132  defects=defectList,
1133  fringes=fringeStruct,
1134  opticsTransmission=opticsTransmission,
1135  filterTransmission=filterTransmission,
1136  sensorTransmission=sensorTransmission,
1137  atmosphereTransmission=atmosphereTransmission,
1138  strayLightData=strayLightData,
1139  illumMaskedImage=illumMaskedImage
1140  )
1141 
1142  @pipeBase.timeMethod
1143  def run(self, ccdExposure, camera=None, bias=None, linearizer=None,
1144  crosstalk=None, crosstalkSources=None,
1145  dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1146  fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1147  sensorTransmission=None, atmosphereTransmission=None,
1148  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1149  isGen3=False,
1150  ):
1151  """!Perform instrument signature removal on an exposure.
1152 
1153  Steps included in the ISR processing, in order performed, are:
1154  - saturation and suspect pixel masking
1155  - overscan subtraction
1156  - CCD assembly of individual amplifiers
1157  - bias subtraction
1158  - variance image construction
1159  - linearization of non-linear response
1160  - crosstalk masking
1161  - brighter-fatter correction
1162  - dark subtraction
1163  - fringe correction
1164  - stray light subtraction
1165  - flat correction
1166  - masking of known defects and camera specific features
1167  - vignette calculation
1168  - appending transmission curve and distortion model
1169 
1170  Parameters
1171  ----------
1172  ccdExposure : `lsst.afw.image.Exposure`
1173  The raw exposure that is to be run through ISR. The
1174  exposure is modified by this method.
1175  camera : `lsst.afw.cameraGeom.Camera`, optional
1176  The camera geometry for this exposure. Required if ``isGen3`` is
1177  `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1178  ``flat`` does not have an associated detector.
1179  bias : `lsst.afw.image.Exposure`, optional
1180  Bias calibration frame.
1181  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1182  Functor for linearization.
1183  crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1184  Calibration for crosstalk.
1185  crosstalkSources : `list`, optional
1186  List of possible crosstalk sources.
1187  dark : `lsst.afw.image.Exposure`, optional
1188  Dark calibration frame.
1189  flat : `lsst.afw.image.Exposure`, optional
1190  Flat calibration frame.
1191  bfKernel : `numpy.ndarray`, optional
1192  Brighter-fatter kernel.
1193  bfGains : `dict` of `float`, optional
1194  Gains used to override the detector's nominal gains for the
1195  brighter-fatter correction. A dict keyed by amplifier name for
1196  the detector in question.
1197  defects : `lsst.meas.algorithms.Defects`, optional
1198  List of defects.
1199  fringes : `lsst.pipe.base.Struct`, optional
1200  Struct containing the fringe correction data, with
1201  elements:
1202  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1203  - ``seed``: random seed derived from the ccdExposureId for random
1204  number generator (`uint32`)
1205  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1206  A ``TransmissionCurve`` that represents the throughput of the optics,
1207  to be evaluated in focal-plane coordinates.
1208  filterTransmission : `lsst.afw.image.TransmissionCurve`
1209  A ``TransmissionCurve`` that represents the throughput of the filter
1210  itself, to be evaluated in focal-plane coordinates.
1211  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1212  A ``TransmissionCurve`` that represents the throughput of the sensor
1213  itself, to be evaluated in post-assembly trimmed detector coordinates.
1214  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1215  A ``TransmissionCurve`` that represents the throughput of the
1216  atmosphere, assumed to be spatially constant.
1217  detectorNum : `int`, optional
1218  The integer number for the detector to process.
1219  isGen3 : bool, optional
1220  Flag this call to run() as using the Gen3 butler environment.
1221  strayLightData : `object`, optional
1222  Opaque object containing calibration information for stray-light
1223  correction. If `None`, no correction will be performed.
1224  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1225  Illumination correction image.
1226 
1227  Returns
1228  -------
1229  result : `lsst.pipe.base.Struct`
1230  Result struct with component:
1231  - ``exposure`` : `afw.image.Exposure`
1232  The fully ISR corrected exposure.
1233  - ``outputExposure`` : `afw.image.Exposure`
1234  An alias for `exposure`
1235  - ``ossThumb`` : `numpy.ndarray`
1236  Thumbnail image of the exposure after overscan subtraction.
1237  - ``flattenedThumb`` : `numpy.ndarray`
1238  Thumbnail image of the exposure after flat-field correction.
1239 
1240  Raises
1241  ------
1242  RuntimeError
1243  Raised if a configuration option is set to True, but the
1244  required calibration data has not been specified.
1245 
1246  Notes
1247  -----
1248  The current processed exposure can be viewed by setting the
1249  appropriate lsstDebug entries in the `debug.display`
1250  dictionary. The names of these entries correspond to some of
1251  the IsrTaskConfig Boolean options, with the value denoting the
1252  frame to use. The exposure is shown inside the matching
1253  option check and after the processing of that step has
1254  finished. The steps with debug points are:
1255 
1256  doAssembleCcd
1257  doBias
1258  doCrosstalk
1259  doBrighterFatter
1260  doDark
1261  doFringe
1262  doStrayLight
1263  doFlat
1264 
1265  In addition, setting the "postISRCCD" entry displays the
1266  exposure after all ISR processing has finished.
1267 
1268  """
1269 
1270  if isGen3 is True:
1271  # Gen3 currently cannot automatically do configuration overrides.
1272  # DM-15257 looks to discuss this issue.
1273  # Configure input exposures;
1274  if detectorNum is None:
1275  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1276 
1277  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1278  bias = self.ensureExposure(bias, camera, detectorNum)
1279  dark = self.ensureExposure(dark, camera, detectorNum)
1280  flat = self.ensureExposure(flat, camera, detectorNum)
1281  else:
1282  if isinstance(ccdExposure, ButlerDataRef):
1283  return self.runDataRef(ccdExposure)
1284 
1285  ccd = ccdExposure.getDetector()
1286  filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1287 
1288  if not ccd:
1289  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1290  ccd = [FakeAmp(ccdExposure, self.config)]
1291 
1292  # Validate Input
1293  if self.config.doBias and bias is None:
1294  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1295  if self.doLinearize(ccd) and linearizer is None:
1296  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1297  if self.config.doBrighterFatter and bfKernel is None:
1298  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1299  if self.config.doDark and dark is None:
1300  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1301  if self.config.doFlat and flat is None:
1302  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1303  if self.config.doDefect and defects is None:
1304  raise RuntimeError("Must supply defects if config.doDefect=True.")
1305  if (self.config.doFringe and filterName in self.fringe.config.filters
1306  and fringes.fringes is None):
1307  # The `fringes` object needs to be a pipeBase.Struct, as
1308  # we use it as a `dict` for the parameters of
1309  # `FringeTask.run()`. The `fringes.fringes` `list` may
1310  # not be `None` if `doFringe=True`. Otherwise, raise.
1311  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1312  if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters
1313  and illumMaskedImage is None):
1314  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1315 
1316  # Begin ISR processing.
1317  if self.config.doConvertIntToFloat:
1318  self.log.info("Converting exposure to floating point values.")
1319  ccdExposure = self.convertIntToFloat(ccdExposure)
1320 
1321  # Amplifier level processing.
1322  overscans = []
1323  for amp in ccd:
1324  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1325  if ccdExposure.getBBox().contains(amp.getBBox()):
1326  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1327  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1328 
1329  if self.config.doOverscan and not badAmp:
1330  # Overscan correction on amp-by-amp basis.
1331  overscanResults = self.overscanCorrection(ccdExposure, amp)
1332  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1333  if overscanResults is not None and \
1334  self.config.qa is not None and self.config.qa.saveStats is True:
1335  if isinstance(overscanResults.overscanFit, float):
1336  qaMedian = overscanResults.overscanFit
1337  qaStdev = float("NaN")
1338  else:
1339  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1340  afwMath.MEDIAN | afwMath.STDEVCLIP)
1341  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1342  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1343 
1344  self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1345  self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1346  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1347  amp.getName(), qaMedian, qaStdev)
1348  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1349  else:
1350  if badAmp:
1351  self.log.warn("Amplifier %s is bad.", amp.getName())
1352  overscanResults = None
1353 
1354  overscans.append(overscanResults if overscanResults is not None else None)
1355  else:
1356  self.log.info("Skipped OSCAN for %s.", amp.getName())
1357 
1358  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1359  self.log.info("Applying crosstalk correction.")
1360  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1361  crosstalkSources=crosstalkSources)
1362  self.debugView(ccdExposure, "doCrosstalk")
1363 
1364  if self.config.doAssembleCcd:
1365  self.log.info("Assembling CCD from amplifiers.")
1366  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1367 
1368  if self.config.expectWcs and not ccdExposure.getWcs():
1369  self.log.warn("No WCS found in input exposure.")
1370  self.debugView(ccdExposure, "doAssembleCcd")
1371 
1372  ossThumb = None
1373  if self.config.qa.doThumbnailOss:
1374  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1375 
1376  if self.config.doBias:
1377  self.log.info("Applying bias correction.")
1378  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1379  trimToFit=self.config.doTrimToMatchCalib)
1380  self.debugView(ccdExposure, "doBias")
1381 
1382  if self.config.doVariance:
1383  for amp, overscanResults in zip(ccd, overscans):
1384  if ccdExposure.getBBox().contains(amp.getBBox()):
1385  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1386  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1387  if overscanResults is not None:
1388  self.updateVariance(ampExposure, amp,
1389  overscanImage=overscanResults.overscanImage)
1390  else:
1391  self.updateVariance(ampExposure, amp,
1392  overscanImage=None)
1393  if self.config.qa is not None and self.config.qa.saveStats is True:
1394  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1395  afwMath.MEDIAN | afwMath.STDEVCLIP)
1396  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1397  qaStats.getValue(afwMath.MEDIAN))
1398  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1399  qaStats.getValue(afwMath.STDEVCLIP))
1400  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1401  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1402  qaStats.getValue(afwMath.STDEVCLIP))
1403 
1404  if self.doLinearize(ccd):
1405  self.log.info("Applying linearizer.")
1406  linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1407  detector=ccd, log=self.log)
1408 
1409  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1410  self.log.info("Applying crosstalk correction.")
1411  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1412  crosstalkSources=crosstalkSources, isTrimmed=True)
1413  self.debugView(ccdExposure, "doCrosstalk")
1414 
1415  # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1416  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1417  if self.config.doDefect:
1418  self.log.info("Masking defects.")
1419  self.maskDefect(ccdExposure, defects)
1420 
1421  if self.config.numEdgeSuspect > 0:
1422  self.log.info("Masking edges as SUSPECT.")
1423  self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1424  maskPlane="SUSPECT")
1425 
1426  if self.config.doNanMasking:
1427  self.log.info("Masking NAN value pixels.")
1428  self.maskNan(ccdExposure)
1429 
1430  if self.config.doWidenSaturationTrails:
1431  self.log.info("Widening saturation trails.")
1432  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1433 
1434  if self.config.doCameraSpecificMasking:
1435  self.log.info("Masking regions for camera specific reasons.")
1436  self.masking.run(ccdExposure)
1437 
1438  if self.config.doBrighterFatter:
1439  # We need to apply flats and darks before we can interpolate, and we
1440  # need to interpolate before we do B-F, but we do B-F without the
1441  # flats and darks applied so we can work in units of electrons or holes.
1442  # This context manager applies and then removes the darks and flats.
1443  #
1444  # We also do not want to interpolate values here, so operate on temporary
1445  # images so we can apply only the BF-correction and roll back the
1446  # interpolation.
1447  interpExp = ccdExposure.clone()
1448  with self.flatContext(interpExp, flat, dark):
1449  isrFunctions.interpolateFromMask(
1450  maskedImage=interpExp.getMaskedImage(),
1451  fwhm=self.config.fwhm,
1452  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1453  maskNameList=self.config.maskListToInterpolate
1454  )
1455  bfExp = interpExp.clone()
1456 
1457  self.log.info("Applying brighter fatter correction using kernel type %s / gains %s.",
1458  type(bfKernel), type(bfGains))
1459  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1460  self.config.brighterFatterMaxIter,
1461  self.config.brighterFatterThreshold,
1462  self.config.brighterFatterApplyGain,
1463  bfGains)
1464  if bfResults[1] == self.config.brighterFatterMaxIter:
1465  self.log.warn("Brighter fatter correction did not converge, final difference %f.",
1466  bfResults[0])
1467  else:
1468  self.log.info("Finished brighter fatter correction in %d iterations.",
1469  bfResults[1])
1470  image = ccdExposure.getMaskedImage().getImage()
1471  bfCorr = bfExp.getMaskedImage().getImage()
1472  bfCorr -= interpExp.getMaskedImage().getImage()
1473  image += bfCorr
1474 
1475  # Applying the brighter-fatter correction applies a
1476  # convolution to the science image. At the edges this
1477  # convolution may not have sufficient valid pixels to
1478  # produce a valid correction. Mark pixels within the size
1479  # of the brighter-fatter kernel as EDGE to warn of this
1480  # fact.
1481  self.log.info("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1482  self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1483  maskPlane="EDGE")
1484 
1485  if self.config.brighterFatterMaskGrowSize > 0:
1486  self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1487  for maskPlane in self.config.maskListToInterpolate:
1488  isrFunctions.growMasks(ccdExposure.getMask(),
1489  radius=self.config.brighterFatterMaskGrowSize,
1490  maskNameList=maskPlane,
1491  maskValue=maskPlane)
1492 
1493  self.debugView(ccdExposure, "doBrighterFatter")
1494 
1495  if self.config.doDark:
1496  self.log.info("Applying dark correction.")
1497  self.darkCorrection(ccdExposure, dark)
1498  self.debugView(ccdExposure, "doDark")
1499 
1500  if self.config.doFringe and not self.config.fringeAfterFlat:
1501  self.log.info("Applying fringe correction before flat.")
1502  self.fringe.run(ccdExposure, **fringes.getDict())
1503  self.debugView(ccdExposure, "doFringe")
1504 
1505  if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1506  self.log.info("Checking strayLight correction.")
1507  self.strayLight.run(ccdExposure, strayLightData)
1508  self.debugView(ccdExposure, "doStrayLight")
1509 
1510  if self.config.doFlat:
1511  self.log.info("Applying flat correction.")
1512  self.flatCorrection(ccdExposure, flat)
1513  self.debugView(ccdExposure, "doFlat")
1514 
1515  if self.config.doApplyGains:
1516  self.log.info("Applying gain correction instead of flat.")
1517  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1519  if self.config.doFringe and self.config.fringeAfterFlat:
1520  self.log.info("Applying fringe correction after flat.")
1521  self.fringe.run(ccdExposure, **fringes.getDict())
1522 
1523  if self.config.doVignette:
1524  self.log.info("Constructing Vignette polygon.")
1525  self.vignettePolygon = self.vignette.run(ccdExposure)
1526 
1527  if self.config.vignette.doWriteVignettePolygon:
1528  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1529 
1530  if self.config.doAttachTransmissionCurve:
1531  self.log.info("Adding transmission curves.")
1532  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1533  filterTransmission=filterTransmission,
1534  sensorTransmission=sensorTransmission,
1535  atmosphereTransmission=atmosphereTransmission)
1536 
1537  flattenedThumb = None
1538  if self.config.qa.doThumbnailFlattened:
1539  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1540 
1541  if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1542  self.log.info("Performing illumination correction.")
1543  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1544  illumMaskedImage, illumScale=self.config.illumScale,
1545  trimToFit=self.config.doTrimToMatchCalib)
1546 
1547  preInterpExp = None
1548  if self.config.doSaveInterpPixels:
1549  preInterpExp = ccdExposure.clone()
1550 
1551  # Reset and interpolate bad pixels.
1552  #
1553  # Large contiguous bad regions (which should have the BAD mask
1554  # bit set) should have their values set to the image median.
1555  # This group should include defects and bad amplifiers. As the
1556  # area covered by these defects are large, there's little
1557  # reason to expect that interpolation would provide a more
1558  # useful value.
1559  #
1560  # Smaller defects can be safely interpolated after the larger
1561  # regions have had their pixel values reset. This ensures
1562  # that the remaining defects adjacent to bad amplifiers (as an
1563  # example) do not attempt to interpolate extreme values.
1564  if self.config.doSetBadRegions:
1565  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1566  if badPixelCount > 0:
1567  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1568 
1569  if self.config.doInterpolate:
1570  self.log.info("Interpolating masked pixels.")
1571  isrFunctions.interpolateFromMask(
1572  maskedImage=ccdExposure.getMaskedImage(),
1573  fwhm=self.config.fwhm,
1574  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1575  maskNameList=list(self.config.maskListToInterpolate)
1576  )
1577 
1578  self.roughZeroPoint(ccdExposure)
1579 
1580  if self.config.doMeasureBackground:
1581  self.log.info("Measuring background level.")
1582  self.measureBackground(ccdExposure, self.config.qa)
1583 
1584  if self.config.qa is not None and self.config.qa.saveStats is True:
1585  for amp in ccd:
1586  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1587  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1588  afwMath.MEDIAN | afwMath.STDEVCLIP)
1589  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1590  qaStats.getValue(afwMath.MEDIAN))
1591  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1592  qaStats.getValue(afwMath.STDEVCLIP))
1593  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1594  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1595  qaStats.getValue(afwMath.STDEVCLIP))
1596 
1597  self.debugView(ccdExposure, "postISRCCD")
1598 
1599  return pipeBase.Struct(
1600  exposure=ccdExposure,
1601  ossThumb=ossThumb,
1602  flattenedThumb=flattenedThumb,
1603 
1604  preInterpolatedExposure=preInterpExp,
1605  outputExposure=ccdExposure,
1606  outputOssThumbnail=ossThumb,
1607  outputFlattenedThumbnail=flattenedThumb,
1608  )
1609 
1610  @pipeBase.timeMethod
1611  def runDataRef(self, sensorRef):
1612  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1613 
1614  This method contains the `CmdLineTask` interface to the ISR
1615  processing. All IO is handled here, freeing the `run()` method
1616  to manage only pixel-level calculations. The steps performed
1617  are:
1618  - Read in necessary detrending/isr/calibration data.
1619  - Process raw exposure in `run()`.
1620  - Persist the ISR-corrected exposure as "postISRCCD" if
1621  config.doWrite=True.
1622 
1623  Parameters
1624  ----------
1625  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1626  DataRef of the detector data to be processed
1627 
1628  Returns
1629  -------
1630  result : `lsst.pipe.base.Struct`
1631  Result struct with component:
1632  - ``exposure`` : `afw.image.Exposure`
1633  The fully ISR corrected exposure.
1634 
1635  Raises
1636  ------
1637  RuntimeError
1638  Raised if a configuration option is set to True, but the
1639  required calibration data does not exist.
1640 
1641  """
1642  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1643 
1644  ccdExposure = sensorRef.get(self.config.datasetType)
1645 
1646  camera = sensorRef.get("camera")
1647  isrData = self.readIsrData(sensorRef, ccdExposure)
1648 
1649  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1650 
1651  if self.config.doWrite:
1652  sensorRef.put(result.exposure, "postISRCCD")
1653  if result.preInterpolatedExposure is not None:
1654  sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1655  if result.ossThumb is not None:
1656  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1657  if result.flattenedThumb is not None:
1658  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1659 
1660  return result
1661 
1662  def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1663  """!Retrieve a calibration dataset for removing instrument signature.
1664 
1665  Parameters
1666  ----------
1667 
1668  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1669  DataRef of the detector data to find calibration datasets
1670  for.
1671  datasetType : `str`
1672  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1673  dateObs : `str`, optional
1674  Date of the observation. Used to correct butler failures
1675  when using fallback filters.
1676  immediate : `Bool`
1677  If True, disable butler proxies to enable error handling
1678  within this routine.
1679 
1680  Returns
1681  -------
1682  exposure : `lsst.afw.image.Exposure`
1683  Requested calibration frame.
1684 
1685  Raises
1686  ------
1687  RuntimeError
1688  Raised if no matching calibration frame can be found.
1689  """
1690  try:
1691  exp = dataRef.get(datasetType, immediate=immediate)
1692  except Exception as exc1:
1693  if not self.config.fallbackFilterName:
1694  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1695  try:
1696  if self.config.useFallbackDate and dateObs:
1697  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1698  dateObs=dateObs, immediate=immediate)
1699  else:
1700  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1701  except Exception as exc2:
1702  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1703  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1704  self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1705 
1706  if self.config.doAssembleIsrExposures:
1707  exp = self.assembleCcd.assembleCcd(exp)
1708  return exp
1709 
1710  def ensureExposure(self, inputExp, camera, detectorNum):
1711  """Ensure that the data returned by Butler is a fully constructed exposure.
1712 
1713  ISR requires exposure-level image data for historical reasons, so if we did
1714  not recieve that from Butler, construct it from what we have, modifying the
1715  input in place.
1716 
1717  Parameters
1718  ----------
1719  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1720  `lsst.afw.image.ImageF`
1721  The input data structure obtained from Butler.
1722  camera : `lsst.afw.cameraGeom.camera`
1723  The camera associated with the image. Used to find the appropriate
1724  detector.
1725  detectorNum : `int`
1726  The detector this exposure should match.
1727 
1728  Returns
1729  -------
1730  inputExp : `lsst.afw.image.Exposure`
1731  The re-constructed exposure, with appropriate detector parameters.
1732 
1733  Raises
1734  ------
1735  TypeError
1736  Raised if the input data cannot be used to construct an exposure.
1737  """
1738  if isinstance(inputExp, afwImage.DecoratedImageU):
1739  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1740  elif isinstance(inputExp, afwImage.ImageF):
1741  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1742  elif isinstance(inputExp, afwImage.MaskedImageF):
1743  inputExp = afwImage.makeExposure(inputExp)
1744  elif isinstance(inputExp, afwImage.Exposure):
1745  pass
1746  elif inputExp is None:
1747  # Assume this will be caught by the setup if it is a problem.
1748  return inputExp
1749  else:
1750  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1751  (type(inputExp), ))
1752 
1753  if inputExp.getDetector() is None:
1754  inputExp.setDetector(camera[detectorNum])
1755 
1756  return inputExp
1757 
1758  def convertIntToFloat(self, exposure):
1759  """Convert exposure image from uint16 to float.
1760 
1761  If the exposure does not need to be converted, the input is
1762  immediately returned. For exposures that are converted to use
1763  floating point pixels, the variance is set to unity and the
1764  mask to zero.
1765 
1766  Parameters
1767  ----------
1768  exposure : `lsst.afw.image.Exposure`
1769  The raw exposure to be converted.
1770 
1771  Returns
1772  -------
1773  newexposure : `lsst.afw.image.Exposure`
1774  The input ``exposure``, converted to floating point pixels.
1775 
1776  Raises
1777  ------
1778  RuntimeError
1779  Raised if the exposure type cannot be converted to float.
1780 
1781  """
1782  if isinstance(exposure, afwImage.ExposureF):
1783  # Nothing to be done
1784  self.log.debug("Exposure already of type float.")
1785  return exposure
1786  if not hasattr(exposure, "convertF"):
1787  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1788 
1789  newexposure = exposure.convertF()
1790  newexposure.variance[:] = 1
1791  newexposure.mask[:] = 0x0
1792 
1793  return newexposure
1794 
1795  def maskAmplifier(self, ccdExposure, amp, defects):
1796  """Identify bad amplifiers, saturated and suspect pixels.
1797 
1798  Parameters
1799  ----------
1800  ccdExposure : `lsst.afw.image.Exposure`
1801  Input exposure to be masked.
1802  amp : `lsst.afw.table.AmpInfoCatalog`
1803  Catalog of parameters defining the amplifier on this
1804  exposure to mask.
1805  defects : `lsst.meas.algorithms.Defects`
1806  List of defects. Used to determine if the entire
1807  amplifier is bad.
1808 
1809  Returns
1810  -------
1811  badAmp : `Bool`
1812  If this is true, the entire amplifier area is covered by
1813  defects and unusable.
1814 
1815  """
1816  maskedImage = ccdExposure.getMaskedImage()
1817 
1818  badAmp = False
1819 
1820  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1821  # comparison with current defects definition.
1822  if defects is not None:
1823  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1824 
1825  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1826  # association with pixels in current ccdExposure).
1827  if badAmp:
1828  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1829  afwImage.PARENT)
1830  maskView = dataView.getMask()
1831  maskView |= maskView.getPlaneBitMask("BAD")
1832  del maskView
1833  return badAmp
1834 
1835  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1836  # Saturation and suspect pixels can be masked now, though.
1837  limits = dict()
1838  if self.config.doSaturation and not badAmp:
1839  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1840  if self.config.doSuspect and not badAmp:
1841  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1842  if math.isfinite(self.config.saturation):
1843  limits.update({self.config.saturatedMaskName: self.config.saturation})
1844 
1845  for maskName, maskThreshold in limits.items():
1846  if not math.isnan(maskThreshold):
1847  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1848  isrFunctions.makeThresholdMask(
1849  maskedImage=dataView,
1850  threshold=maskThreshold,
1851  growFootprints=0,
1852  maskName=maskName
1853  )
1854 
1855  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1856  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1857  afwImage.PARENT)
1858  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1859  self.config.suspectMaskName])
1860  if numpy.all(maskView.getArray() & maskVal > 0):
1861  badAmp = True
1862  maskView |= maskView.getPlaneBitMask("BAD")
1863 
1864  return badAmp
1865 
1866  def overscanCorrection(self, ccdExposure, amp):
1867  """Apply overscan correction in place.
1868 
1869  This method does initial pixel rejection of the overscan
1870  region. The overscan can also be optionally segmented to
1871  allow for discontinuous overscan responses to be fit
1872  separately. The actual overscan subtraction is performed by
1873  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1874  which is called here after the amplifier is preprocessed.
1875 
1876  Parameters
1877  ----------
1878  ccdExposure : `lsst.afw.image.Exposure`
1879  Exposure to have overscan correction performed.
1880  amp : `lsst.afw.table.AmpInfoCatalog`
1881  The amplifier to consider while correcting the overscan.
1882 
1883  Returns
1884  -------
1885  overscanResults : `lsst.pipe.base.Struct`
1886  Result struct with components:
1887  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1888  Value or fit subtracted from the amplifier image data.
1889  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1890  Value or fit subtracted from the overscan image data.
1891  - ``overscanImage`` : `lsst.afw.image.Image`
1892  Image of the overscan region with the overscan
1893  correction applied. This quantity is used to estimate
1894  the amplifier read noise empirically.
1895 
1896  Raises
1897  ------
1898  RuntimeError
1899  Raised if the ``amp`` does not contain raw pixel information.
1900 
1901  See Also
1902  --------
1903  lsst.ip.isr.isrFunctions.overscanCorrection
1904  """
1905  if amp.getRawHorizontalOverscanBBox().isEmpty():
1906  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1907  return None
1908 
1909  statControl = afwMath.StatisticsControl()
1910  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1911 
1912  # Determine the bounding boxes
1913  dataBBox = amp.getRawDataBBox()
1914  oscanBBox = amp.getRawHorizontalOverscanBBox()
1915  dx0 = 0
1916  dx1 = 0
1917 
1918  prescanBBox = amp.getRawPrescanBBox()
1919  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1920  dx0 += self.config.overscanNumLeadingColumnsToSkip
1921  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1922  else:
1923  dx0 += self.config.overscanNumTrailingColumnsToSkip
1924  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1925 
1926  # Determine if we need to work on subregions of the amplifier and overscan.
1927  imageBBoxes = []
1928  overscanBBoxes = []
1929 
1930  if ((self.config.overscanBiasJump
1931  and self.config.overscanBiasJumpLocation)
1932  and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
1933  and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1934  self.config.overscanBiasJumpDevices)):
1935  if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
1936  yLower = self.config.overscanBiasJumpLocation
1937  yUpper = dataBBox.getHeight() - yLower
1938  else:
1939  yUpper = self.config.overscanBiasJumpLocation
1940  yLower = dataBBox.getHeight() - yUpper
1941 
1942  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1943  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1944  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1945  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1946  yLower)))
1947 
1948  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1949  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1950  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1951  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1952  yUpper)))
1953  else:
1954  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1955  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1956  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1957  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1958  oscanBBox.getHeight())))
1959 
1960  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1961  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1962  ampImage = ccdExposure.maskedImage[imageBBox]
1963  overscanImage = ccdExposure.maskedImage[overscanBBox]
1964 
1965  overscanArray = overscanImage.image.array
1966  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1967  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1968  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1969 
1970  statControl = afwMath.StatisticsControl()
1971  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1972 
1973  overscanResults = self.overscan.run(ampImage.getImage(), overscanImage)
1974 
1975  # Measure average overscan levels and record them in the metadata.
1976  levelStat = afwMath.MEDIAN
1977  sigmaStat = afwMath.STDEVCLIP
1978 
1979  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1980  self.config.qa.flatness.nIter)
1981  metadata = ccdExposure.getMetadata()
1982  ampNum = amp.getName()
1983  # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1984  if isinstance(overscanResults.overscanFit, float):
1985  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1986  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1987  else:
1988  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1989  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1990  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1991 
1992  return overscanResults
1993 
1994  def updateVariance(self, ampExposure, amp, overscanImage=None):
1995  """Set the variance plane using the amplifier gain and read noise
1996 
1997  The read noise is calculated from the ``overscanImage`` if the
1998  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1999  the value from the amplifier data is used.
2000 
2001  Parameters
2002  ----------
2003  ampExposure : `lsst.afw.image.Exposure`
2004  Exposure to process.
2005  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2006  Amplifier detector data.
2007  overscanImage : `lsst.afw.image.MaskedImage`, optional.
2008  Image of overscan, required only for empirical read noise.
2009 
2010  See also
2011  --------
2012  lsst.ip.isr.isrFunctions.updateVariance
2013  """
2014  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2015  gain = amp.getGain()
2016 
2017  if math.isnan(gain):
2018  gain = 1.0
2019  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2020  elif gain <= 0:
2021  patchedGain = 1.0
2022  self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
2023  amp.getName(), gain, patchedGain)
2024  gain = patchedGain
2025 
2026  if self.config.doEmpiricalReadNoise and overscanImage is None:
2027  self.log.info("Overscan is none for EmpiricalReadNoise.")
2028 
2029  if self.config.doEmpiricalReadNoise and overscanImage is not None:
2030  stats = afwMath.StatisticsControl()
2031  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2032  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2033  self.log.info("Calculated empirical read noise for amp %s: %f.",
2034  amp.getName(), readNoise)
2035  else:
2036  readNoise = amp.getReadNoise()
2037 
2038  isrFunctions.updateVariance(
2039  maskedImage=ampExposure.getMaskedImage(),
2040  gain=gain,
2041  readNoise=readNoise,
2042  )
2043 
2044  def darkCorrection(self, exposure, darkExposure, invert=False):
2045  """!Apply dark correction in place.
2046 
2047  Parameters
2048  ----------
2049  exposure : `lsst.afw.image.Exposure`
2050  Exposure to process.
2051  darkExposure : `lsst.afw.image.Exposure`
2052  Dark exposure of the same size as ``exposure``.
2053  invert : `Bool`, optional
2054  If True, re-add the dark to an already corrected image.
2055 
2056  Raises
2057  ------
2058  RuntimeError
2059  Raised if either ``exposure`` or ``darkExposure`` do not
2060  have their dark time defined.
2061 
2062  See Also
2063  --------
2064  lsst.ip.isr.isrFunctions.darkCorrection
2065  """
2066  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2067  if math.isnan(expScale):
2068  raise RuntimeError("Exposure darktime is NAN.")
2069  if darkExposure.getInfo().getVisitInfo() is not None \
2070  and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2071  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2072  else:
2073  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
2074  # so getDarkTime() does not exist.
2075  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2076  darkScale = 1.0
2077 
2078  isrFunctions.darkCorrection(
2079  maskedImage=exposure.getMaskedImage(),
2080  darkMaskedImage=darkExposure.getMaskedImage(),
2081  expScale=expScale,
2082  darkScale=darkScale,
2083  invert=invert,
2084  trimToFit=self.config.doTrimToMatchCalib
2085  )
2086 
2087  def doLinearize(self, detector):
2088  """!Check if linearization is needed for the detector cameraGeom.
2089 
2090  Checks config.doLinearize and the linearity type of the first
2091  amplifier.
2092 
2093  Parameters
2094  ----------
2095  detector : `lsst.afw.cameraGeom.Detector`
2096  Detector to get linearity type from.
2097 
2098  Returns
2099  -------
2100  doLinearize : `Bool`
2101  If True, linearization should be performed.
2102  """
2103  return self.config.doLinearize and \
2104  detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2105 
2106  def flatCorrection(self, exposure, flatExposure, invert=False):
2107  """!Apply flat correction in place.
2108 
2109  Parameters
2110  ----------
2111  exposure : `lsst.afw.image.Exposure`
2112  Exposure to process.
2113  flatExposure : `lsst.afw.image.Exposure`
2114  Flat exposure of the same size as ``exposure``.
2115  invert : `Bool`, optional
2116  If True, unflatten an already flattened image.
2117 
2118  See Also
2119  --------
2120  lsst.ip.isr.isrFunctions.flatCorrection
2121  """
2122  isrFunctions.flatCorrection(
2123  maskedImage=exposure.getMaskedImage(),
2124  flatMaskedImage=flatExposure.getMaskedImage(),
2125  scalingType=self.config.flatScalingType,
2126  userScale=self.config.flatUserScale,
2127  invert=invert,
2128  trimToFit=self.config.doTrimToMatchCalib
2129  )
2130 
2131  def saturationDetection(self, exposure, amp):
2132  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2133 
2134  Parameters
2135  ----------
2136  exposure : `lsst.afw.image.Exposure`
2137  Exposure to process. Only the amplifier DataSec is processed.
2138  amp : `lsst.afw.table.AmpInfoCatalog`
2139  Amplifier detector data.
2140 
2141  See Also
2142  --------
2143  lsst.ip.isr.isrFunctions.makeThresholdMask
2144  """
2145  if not math.isnan(amp.getSaturation()):
2146  maskedImage = exposure.getMaskedImage()
2147  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2148  isrFunctions.makeThresholdMask(
2149  maskedImage=dataView,
2150  threshold=amp.getSaturation(),
2151  growFootprints=0,
2152  maskName=self.config.saturatedMaskName,
2153  )
2154 
2155  def saturationInterpolation(self, exposure):
2156  """!Interpolate over saturated pixels, in place.
2157 
2158  This method should be called after `saturationDetection`, to
2159  ensure that the saturated pixels have been identified in the
2160  SAT mask. It should also be called after `assembleCcd`, since
2161  saturated regions may cross amplifier boundaries.
2162 
2163  Parameters
2164  ----------
2165  exposure : `lsst.afw.image.Exposure`
2166  Exposure to process.
2167 
2168  See Also
2169  --------
2170  lsst.ip.isr.isrTask.saturationDetection
2171  lsst.ip.isr.isrFunctions.interpolateFromMask
2172  """
2173  isrFunctions.interpolateFromMask(
2174  maskedImage=exposure.getMaskedImage(),
2175  fwhm=self.config.fwhm,
2176  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2177  maskNameList=list(self.config.saturatedMaskName),
2178  )
2179 
2180  def suspectDetection(self, exposure, amp):
2181  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2182 
2183  Parameters
2184  ----------
2185  exposure : `lsst.afw.image.Exposure`
2186  Exposure to process. Only the amplifier DataSec is processed.
2187  amp : `lsst.afw.table.AmpInfoCatalog`
2188  Amplifier detector data.
2189 
2190  See Also
2191  --------
2192  lsst.ip.isr.isrFunctions.makeThresholdMask
2193 
2194  Notes
2195  -----
2196  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2197  This is intended to indicate pixels that may be affected by unknown systematics;
2198  for example if non-linearity corrections above a certain level are unstable
2199  then that would be a useful value for suspectLevel. A value of `nan` indicates
2200  that no such level exists and no pixels are to be masked as suspicious.
2201  """
2202  suspectLevel = amp.getSuspectLevel()
2203  if math.isnan(suspectLevel):
2204  return
2205 
2206  maskedImage = exposure.getMaskedImage()
2207  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2208  isrFunctions.makeThresholdMask(
2209  maskedImage=dataView,
2210  threshold=suspectLevel,
2211  growFootprints=0,
2212  maskName=self.config.suspectMaskName,
2213  )
2214 
2215  def maskDefect(self, exposure, defectBaseList):
2216  """!Mask defects using mask plane "BAD", in place.
2217 
2218  Parameters
2219  ----------
2220  exposure : `lsst.afw.image.Exposure`
2221  Exposure to process.
2222  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2223  `lsst.afw.image.DefectBase`.
2224  List of defects to mask.
2225 
2226  Notes
2227  -----
2228  Call this after CCD assembly, since defects may cross amplifier boundaries.
2229  """
2230  maskedImage = exposure.getMaskedImage()
2231  if not isinstance(defectBaseList, Defects):
2232  # Promotes DefectBase to Defect
2233  defectList = Defects(defectBaseList)
2234  else:
2235  defectList = defectBaseList
2236  defectList.maskPixels(maskedImage, maskName="BAD")
2237 
2238  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"):
2239  """!Mask edge pixels with applicable mask plane.
2240 
2241  Parameters
2242  ----------
2243  exposure : `lsst.afw.image.Exposure`
2244  Exposure to process.
2245  numEdgePixels : `int`, optional
2246  Number of edge pixels to mask.
2247  maskPlane : `str`, optional
2248  Mask plane name to use.
2249  """
2250  maskedImage = exposure.getMaskedImage()
2251  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2252 
2253  if numEdgePixels > 0:
2254  goodBBox = maskedImage.getBBox()
2255  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2256  goodBBox.grow(-numEdgePixels)
2257  # Mask pixels outside goodBBox
2258  SourceDetectionTask.setEdgeBits(
2259  maskedImage,
2260  goodBBox,
2261  maskBitMask
2262  )
2263 
2264  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2265  """Mask and interpolate defects using mask plane "BAD", in place.
2266 
2267  Parameters
2268  ----------
2269  exposure : `lsst.afw.image.Exposure`
2270  Exposure to process.
2271  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2272  `lsst.afw.image.DefectBase`.
2273  List of defects to mask and interpolate.
2274 
2275  See Also
2276  --------
2277  lsst.ip.isr.isrTask.maskDefect()
2278  """
2279  self.maskDefect(exposure, defectBaseList)
2280  self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2281  maskPlane="SUSPECT")
2282  isrFunctions.interpolateFromMask(
2283  maskedImage=exposure.getMaskedImage(),
2284  fwhm=self.config.fwhm,
2285  growSaturatedFootprints=0,
2286  maskNameList=["BAD"],
2287  )
2288 
2289  def maskNan(self, exposure):
2290  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2291 
2292  Parameters
2293  ----------
2294  exposure : `lsst.afw.image.Exposure`
2295  Exposure to process.
2296 
2297  Notes
2298  -----
2299  We mask over all NaNs, including those that are masked with
2300  other bits (because those may or may not be interpolated over
2301  later, and we want to remove all NaNs). Despite this
2302  behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2303  the historical name.
2304  """
2305  maskedImage = exposure.getMaskedImage()
2306 
2307  # Find and mask NaNs
2308  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2309  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2310  numNans = maskNans(maskedImage, maskVal)
2311  self.metadata.set("NUMNANS", numNans)
2312  if numNans > 0:
2313  self.log.warn("There were %d unmasked NaNs.", numNans)
2314 
2315  def maskAndInterpolateNan(self, exposure):
2316  """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2317 
2318  Parameters
2319  ----------
2320  exposure : `lsst.afw.image.Exposure`
2321  Exposure to process.
2322 
2323  See Also
2324  --------
2325  lsst.ip.isr.isrTask.maskNan()
2326  """
2327  self.maskNan(exposure)
2328  isrFunctions.interpolateFromMask(
2329  maskedImage=exposure.getMaskedImage(),
2330  fwhm=self.config.fwhm,
2331  growSaturatedFootprints=0,
2332  maskNameList=["UNMASKEDNAN"],
2333  )
2334 
2335  def measureBackground(self, exposure, IsrQaConfig=None):
2336  """Measure the image background in subgrids, for quality control purposes.
2337 
2338  Parameters
2339  ----------
2340  exposure : `lsst.afw.image.Exposure`
2341  Exposure to process.
2342  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2343  Configuration object containing parameters on which background
2344  statistics and subgrids to use.
2345  """
2346  if IsrQaConfig is not None:
2347  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2348  IsrQaConfig.flatness.nIter)
2349  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2350  statsControl.setAndMask(maskVal)
2351  maskedImage = exposure.getMaskedImage()
2352  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2353  skyLevel = stats.getValue(afwMath.MEDIAN)
2354  skySigma = stats.getValue(afwMath.STDEVCLIP)
2355  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2356  metadata = exposure.getMetadata()
2357  metadata.set('SKYLEVEL', skyLevel)
2358  metadata.set('SKYSIGMA', skySigma)
2359 
2360  # calcluating flatlevel over the subgrids
2361  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2362  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2363  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2364  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2365  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2366  skyLevels = numpy.zeros((nX, nY))
2367 
2368  for j in range(nY):
2369  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2370  for i in range(nX):
2371  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2372 
2373  xLLC = xc - meshXHalf
2374  yLLC = yc - meshYHalf
2375  xURC = xc + meshXHalf - 1
2376  yURC = yc + meshYHalf - 1
2377 
2378  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2379  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2380 
2381  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2382 
2383  good = numpy.where(numpy.isfinite(skyLevels))
2384  skyMedian = numpy.median(skyLevels[good])
2385  flatness = (skyLevels[good] - skyMedian) / skyMedian
2386  flatness_rms = numpy.std(flatness)
2387  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2388 
2389  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2390  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2391  nX, nY, flatness_pp, flatness_rms)
2392 
2393  metadata.set('FLATNESS_PP', float(flatness_pp))
2394  metadata.set('FLATNESS_RMS', float(flatness_rms))
2395  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2396  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2397  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2398 
2399  def roughZeroPoint(self, exposure):
2400  """Set an approximate magnitude zero point for the exposure.
2401 
2402  Parameters
2403  ----------
2404  exposure : `lsst.afw.image.Exposure`
2405  Exposure to process.
2406  """
2407  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2408  if filterName in self.config.fluxMag0T1:
2409  fluxMag0 = self.config.fluxMag0T1[filterName]
2410  else:
2411  self.log.warn("No rough magnitude zero point set for filter %s.", filterName)
2412  fluxMag0 = self.config.defaultFluxMag0T1
2413 
2414  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2415  if not expTime > 0: # handle NaN as well as <= 0
2416  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2417  return
2418 
2419  self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2420  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2421 
2422  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2423  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2424 
2425  Parameters
2426  ----------
2427  ccdExposure : `lsst.afw.image.Exposure`
2428  Exposure to process.
2429  fpPolygon : `lsst.afw.geom.Polygon`
2430  Polygon in focal plane coordinates.
2431  """
2432  # Get ccd corners in focal plane coordinates
2433  ccd = ccdExposure.getDetector()
2434  fpCorners = ccd.getCorners(FOCAL_PLANE)
2435  ccdPolygon = Polygon(fpCorners)
2436 
2437  # Get intersection of ccd corners with fpPolygon
2438  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2439 
2440  # Transform back to pixel positions and build new polygon
2441  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2442  validPolygon = Polygon(ccdPoints)
2443  ccdExposure.getInfo().setValidPolygon(validPolygon)
2444 
2445  @contextmanager
2446  def flatContext(self, exp, flat, dark=None):
2447  """Context manager that applies and removes flats and darks,
2448  if the task is configured to apply them.
2449 
2450  Parameters
2451  ----------
2452  exp : `lsst.afw.image.Exposure`
2453  Exposure to process.
2454  flat : `lsst.afw.image.Exposure`
2455  Flat exposure the same size as ``exp``.
2456  dark : `lsst.afw.image.Exposure`, optional
2457  Dark exposure the same size as ``exp``.
2458 
2459  Yields
2460  ------
2461  exp : `lsst.afw.image.Exposure`
2462  The flat and dark corrected exposure.
2463  """
2464  if self.config.doDark and dark is not None:
2465  self.darkCorrection(exp, dark)
2466  if self.config.doFlat:
2467  self.flatCorrection(exp, flat)
2468  try:
2469  yield exp
2470  finally:
2471  if self.config.doFlat:
2472  self.flatCorrection(exp, flat, invert=True)
2473  if self.config.doDark and dark is not None:
2474  self.darkCorrection(exp, dark, invert=True)
2475 
2476  def debugView(self, exposure, stepname):
2477  """Utility function to examine ISR exposure at different stages.
2478 
2479  Parameters
2480  ----------
2481  exposure : `lsst.afw.image.Exposure`
2482  Exposure to view.
2483  stepname : `str`
2484  State of processing to view.
2485  """
2486  frame = getDebugFrame(self._display, stepname)
2487  if frame:
2488  display = getDisplay(frame)
2489  display.scale('asinh', 'zscale')
2490  display.mtv(exposure)
2491  prompt = "Press Enter to continue [c]... "
2492  while True:
2493  ans = input(prompt).lower()
2494  if ans in ("", "c",):
2495  break
2496 
2497 
2498 class FakeAmp(object):
2499  """A Detector-like object that supports returning gain and saturation level
2500 
2501  This is used when the input exposure does not have a detector.
2502 
2503  Parameters
2504  ----------
2505  exposure : `lsst.afw.image.Exposure`
2506  Exposure to generate a fake amplifier for.
2507  config : `lsst.ip.isr.isrTaskConfig`
2508  Configuration to apply to the fake amplifier.
2509  """
2510 
2511  def __init__(self, exposure, config):
2512  self._bbox = exposure.getBBox(afwImage.LOCAL)
2514  self._gain = config.gain
2515  self._readNoise = config.readNoise
2516  self._saturation = config.saturation
2517 
2518  def getBBox(self):
2519  return self._bbox
2520 
2521  def getRawBBox(self):
2522  return self._bbox
2523 
2525  return self._RawHorizontalOverscanBBox
2526 
2527  def getGain(self):
2528  return self._gain
2529 
2530  def getReadNoise(self):
2531  return self._readNoise
2532 
2533  def getSaturation(self):
2534  return self._saturation
2535 
2536  def getSuspectLevel(self):
2537  return float("NaN")
2538 
2539 
2540 class RunIsrConfig(pexConfig.Config):
2541  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2542 
2543 
2544 class RunIsrTask(pipeBase.CmdLineTask):
2545  """Task to wrap the default IsrTask to allow it to be retargeted.
2546 
2547  The standard IsrTask can be called directly from a command line
2548  program, but doing so removes the ability of the task to be
2549  retargeted. As most cameras override some set of the IsrTask
2550  methods, this would remove those data-specific methods in the
2551  output post-ISR images. This wrapping class fixes the issue,
2552  allowing identical post-ISR images to be generated by both the
2553  processCcd and isrTask code.
2554  """
2555  ConfigClass = RunIsrConfig
2556  _DefaultName = "runIsr"
2557 
2558  def __init__(self, *args, **kwargs):
2559  super().__init__(*args, **kwargs)
2560  self.makeSubtask("isr")
2561 
2562  def runDataRef(self, dataRef):
2563  """
2564  Parameters
2565  ----------
2566  dataRef : `lsst.daf.persistence.ButlerDataRef`
2567  data reference of the detector data to be processed
2568 
2569  Returns
2570  -------
2571  result : `pipeBase.Struct`
2572  Result struct with component:
2573 
2574  - exposure : `lsst.afw.image.Exposure`
2575  Post-ISR processed exposure.
2576  """
2577  return self.isr.runDataRef(dataRef)
lsst::ip::isr.isrTask.IsrTaskConfig.maskListToInterpolate
maskListToInterpolate
Definition: isrTask.py:742
lsst::ip::isr.isrTask.FakeAmp.getRawBBox
def getRawBBox(self)
Definition: isrTask.py:2521
lsst::afw::image
lsst::ip::isr.linearize.Linearizer
Definition: linearize.py:40
lsst::meas::algorithms::detection
lsst::ip::isr.isrTask.IsrTask.debugView
def debugView(self, exposure, stepname)
Definition: isrTask.py:2476
lsst::ip::isr.isrTask.FakeAmp.getReadNoise
def getReadNoise(self)
Definition: isrTask.py:2530
lsst::ip::isr.isrTask.IsrTask.maskAndInterpolateNan
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2315
lsst::ip::isr.isrTask.IsrTaskConfig.validate
def validate(self)
Definition: isrTask.py:834
lsst::ip::isr.isrTask.IsrTaskConnections
Definition: isrTask.py:103
lsst::ip::isr.isrTask.FakeAmp.getRawHorizontalOverscanBBox
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2524
lsst::ip::isr.isrTask.FakeAmp._RawHorizontalOverscanBBox
_RawHorizontalOverscanBBox
Definition: isrTask.py:2513
lsst::ip::isr.isrTask.IsrTask.measureBackground
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2335
lsst::ip::isr.isrTask.IsrTask.maskAmplifier
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1795
lsst::ip::isr.crosstalk.CrosstalkCalib
Definition: crosstalk.py:40
lsst::ip::isr.isrTask.IsrTask.roughZeroPoint
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2399
lsst::ip::isr.isrTask.crosstalkSourceLookup
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
Definition: isrTask.py:62
lsst::ip::isr.isrTask.IsrTask.saturationDetection
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
Definition: isrTask.py:2131
lsst::ip::isr.isrTask.IsrTask.maskNan
def maskNan(self, exposure)
Definition: isrTask.py:2289
lsst::ip::isr.isrTask.IsrTaskConfig.doNanInterpolation
doNanInterpolation
Definition: isrTask.py:729
lsst::ip::isr.isrTask.IsrTask.runDataRef
def runDataRef(self, sensorRef)
Definition: isrTask.py:1611
lsst::afw::display
lsst::ip::isr.isrTask.IsrTask.suspectDetection
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:2180
lsst::ip::isr.isrTask.IsrTask.maskEdges
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT")
Mask edge pixels with applicable mask plane.
Definition: isrTask.py:2238
lsst::ip::isr.isrTask.FakeAmp.getBBox
def getBBox(self)
Definition: isrTask.py:2518
lsst::ip::isr.isrTask.FakeAmp
Definition: isrTask.py:2498
lsst::ip::isr.isrTask.IsrTask.maskDefect
def maskDefect(self, exposure, defectBaseList)
Mask defects using mask plane "BAD", in place.
Definition: isrTask.py:2215
lsst::ip::isr.isrTask.FakeAmp._readNoise
_readNoise
Definition: isrTask.py:2515
lsst::ip::isr.isrTask.IsrTaskConfig
Definition: isrTask.py:287
lsst::ip::isr.isrTask.IsrTask.flatCorrection
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:2106
lsst::ip::isr.isrQa.IsrQaConfig
Definition: isrQa.py:58
lsst::ip::isr.isrTask.FakeAmp.getSuspectLevel
def getSuspectLevel(self)
Definition: isrTask.py:2536
lsst::ip::isr.isrTask.IsrTask
Definition: isrTask.py:844
lsst::ip::isr.isrTask.IsrTask.overscanCorrection
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1866
lsst::ip::isr.isrTask.RunIsrTask
Definition: isrTask.py:2544
lsst::ip::isr.isrTask.IsrTask.doLinearize
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:2087
lsst::ip::isr.isrTask.IsrTaskConfig.doFlat
doFlat
Definition: isrTask.py:638
lsst::ip::isr.isrTask.IsrTask.__init__
def __init__(self, **kwargs)
Definition: isrTask.py:875
lsst::ip::isr::maskNans
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
lsst::ip::isr.isrTask.IsrTask.darkCorrection
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:2044
lsst::ip::isr.isrTask.IsrTask.readIsrData
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:968
lsst::ip::isr.isrTask.FakeAmp._gain
_gain
Definition: isrTask.py:2514
lsst::ip::isr.isrTask.IsrTask.convertIntToFloat
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1758
lsst::ip::isr.isrTask.IsrTask.flatContext
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2446
lsst::ip::isr.isrTask.RunIsrTask.__init__
def __init__(self, *args, **kwargs)
Definition: isrTask.py:2558
lsst::ip::isr.isrTask.IsrTaskConfig.doSaturationInterpolation
doSaturationInterpolation
Definition: isrTask.py:722
lsst::ip::isr.isrTask.FakeAmp._bbox
_bbox
Definition: isrTask.py:2512
lsst::daf::persistence::butler
lsst::ip::isr.isrTask.IsrTaskConfig.doApplyGains
doApplyGains
Definition: isrTask.py:670
lsst::ip::isr.isrTask.IsrTask.vignettePolygon
vignettePolygon
Definition: isrTask.py:1518
lsst::ip::isr.isrTask.IsrTask.updateVariance
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1994
lsst::ip::isr.isrTask.IsrTask.maskAndInterpolateDefects
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2264
lsst::ip::isr.isrTask.FakeAmp._saturation
_saturation
Definition: isrTask.py:2516
lsst::ip::isr.isrTask.FakeAmp.__init__
def __init__(self, exposure, config)
Definition: isrTask.py:2511
lsst::ip::isr.isrTask.RunIsrTask.runDataRef
def runDataRef(self, dataRef)
Definition: isrTask.py:2562
lsst::ip::isr.isrTask.FakeAmp.getGain
def getGain(self)
Definition: isrTask.py:2527
lsst::geom
lsst::ip::isr.isrTask.IsrTask.ensureExposure
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1710
lsst::afw::cameraGeom
lsst::ip::isr.isrTask.IsrTask.setValidPolygonIntersect
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2422
lsst::afw::math
Point< int, 2 >
lsst::geom::Box2I
lsst::ip::isr.isrTask.FakeAmp.getSaturation
def getSaturation(self)
Definition: isrTask.py:2533
lsst::ip::isr.isrTask.IsrTask.saturationInterpolation
def saturationInterpolation(self, exposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:2155
lsst::ip::isr.isrTask.IsrTask.run
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:1143
lsst::daf::persistence
lsst::ip::isr.isrTask.IsrTask.runQuantum
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:885
lsst::ip::isr.isrTask.IsrTask.getIsrExposure
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1662
lsst::pipe::base
lsst::meas::algorithms
lsst::ip::isr.isrTask.IsrTaskConnections.__init__
def __init__(self, *config=None)
Definition: isrTask.py:238
Extent< int, 2 >
lsst::pipe::base::connectionTypes
lsst::afw::geom
lsst::ip::isr.isrTask.RunIsrConfig
Definition: isrTask.py:2540