lsst.ip.isr  20.0.0-5-gfcb36e1
isrTask.py
Go to the documentation of this file.
1 # This file is part of ip_isr.
2 #
3 # Developed for the LSST Data Management System.
4 # This product includes software developed by the LSST Project
5 # (https://www.lsst.org).
6 # See the COPYRIGHT file at the top-level directory of this distribution
7 # for details of code ownership.
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the GNU General Public License
20 # along with this program. If not, see <https://www.gnu.org/licenses/>.
21 
22 import math
23 import numpy
24 
25 import lsst.geom
26 import lsst.afw.image as afwImage
27 import lsst.afw.math as afwMath
28 import lsst.pex.config as pexConfig
29 import lsst.pipe.base as pipeBase
31 
32 from contextlib import contextmanager
33 from lsstDebug import getDebugFrame
34 
35 from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36  ReadoutCorner)
37 from lsst.afw.display import getDisplay
38 from lsst.afw.geom import Polygon
39 from lsst.daf.persistence import ButlerDataRef
40 from lsst.daf.persistence.butler import NoResults
41 from lsst.meas.algorithms.detection import SourceDetectionTask
42 from lsst.meas.algorithms import Defects
43 
44 from . import isrFunctions
45 from . import isrQa
46 from . import linearize
47 
48 from .assembleCcdTask import AssembleCcdTask
49 from .crosstalk import CrosstalkTask, CrosstalkCalib
50 from .fringe import FringeTask
51 from .isr import maskNans
52 from .masking import MaskingTask
53 from .overscan import OverscanCorrectionTask
54 from .straylight import StrayLightTask
55 from .vignette import VignetteTask
56 from lsst.daf.butler import DimensionGraph
57 
58 
59 __all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
60 
61 
62 def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections):
63  """Lookup function to identify crosstalkSource entries.
64 
65  This should return an empty list under most circumstances. Only
66  when inter-chip crosstalk has been identified should this be
67  populated.
68 
69  This will be unused until DM-25348 resolves the quantum graph
70  generation issue.
71 
72  Parameters
73  ----------
74  datasetType : `str`
75  Dataset to lookup.
76  registry : `lsst.daf.butler.Registry`
77  Butler registry to query.
78  quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79  Data id to transform to identify crosstalkSources. The
80  ``detector`` entry will be stripped.
81  collections : `lsst.daf.butler.CollectionSearch`
82  Collections to search through.
83 
84  Returns
85  -------
86  results : `list` [`lsst.daf.butler.DatasetRef`]
87  List of datasets that match the query that will be used as
88  crosstalkSources.
89  """
90  newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"]))
91  results = list(registry.queryDatasets(datasetType,
92  collections=collections,
93  dataId=newDataId,
94  deduplicate=True,
95  expand=True))
96  return results
97 
98 
99 class IsrTaskConnections(pipeBase.PipelineTaskConnections,
100  dimensions={"instrument", "exposure", "detector"},
101  defaultTemplates={}):
102  ccdExposure = cT.Input(
103  name="raw",
104  doc="Input exposure to process.",
105  storageClass="Exposure",
106  dimensions=["instrument", "exposure", "detector"],
107  )
108  camera = cT.PrerequisiteInput(
109  name="camera",
110  storageClass="Camera",
111  doc="Input camera to construct complete exposures.",
112  dimensions=["instrument", "calibration_label"],
113  )
114 
115  crosstalk = cT.PrerequisiteInput(
116  name="crosstalk",
117  doc="Input crosstalk object",
118  storageClass="CrosstalkCalib",
119  dimensions=["instrument", "calibration_label", "detector"],
120  )
121  # TODO: DM-25348. This does not work yet to correctly load
122  # possible crosstalk sources.
123  crosstalkSources = cT.PrerequisiteInput(
124  name="isrOverscanCorrected",
125  doc="Overscan corrected input images.",
126  storageClass="Exposure",
127  dimensions=["instrument", "exposure", "detector"],
128  deferLoad=True,
129  multiple=True,
130  lookupFunction=crosstalkSourceLookup,
131  )
132  bias = cT.PrerequisiteInput(
133  name="bias",
134  doc="Input bias calibration.",
135  storageClass="ExposureF",
136  dimensions=["instrument", "calibration_label", "detector"],
137  )
138  dark = cT.PrerequisiteInput(
139  name='dark',
140  doc="Input dark calibration.",
141  storageClass="ExposureF",
142  dimensions=["instrument", "calibration_label", "detector"],
143  )
144  flat = cT.PrerequisiteInput(
145  name="flat",
146  doc="Input flat calibration.",
147  storageClass="ExposureF",
148  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
149  )
150  fringes = cT.PrerequisiteInput(
151  name="fringe",
152  doc="Input fringe calibration.",
153  storageClass="ExposureF",
154  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
155  )
156  strayLightData = cT.PrerequisiteInput(
157  name='yBackground',
158  doc="Input stray light calibration.",
159  storageClass="StrayLightData",
160  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
161  )
162  bfKernel = cT.PrerequisiteInput(
163  name='bfKernel',
164  doc="Input brighter-fatter kernel.",
165  storageClass="NumpyArray",
166  dimensions=["instrument", "calibration_label"],
167  )
168  newBFKernel = cT.PrerequisiteInput(
169  name='brighterFatterKernel',
170  doc="Newer complete kernel + gain solutions.",
171  storageClass="BrighterFatterKernel",
172  dimensions=["instrument", "calibration_label", "detector"],
173  )
174  defects = cT.PrerequisiteInput(
175  name='defects',
176  doc="Input defect tables.",
177  storageClass="Defects",
178  dimensions=["instrument", "calibration_label", "detector"],
179  )
180  opticsTransmission = cT.PrerequisiteInput(
181  name="transmission_optics",
182  storageClass="TransmissionCurve",
183  doc="Transmission curve due to the optics.",
184  dimensions=["instrument", "calibration_label"],
185  )
186  filterTransmission = cT.PrerequisiteInput(
187  name="transmission_filter",
188  storageClass="TransmissionCurve",
189  doc="Transmission curve due to the filter.",
190  dimensions=["instrument", "physical_filter", "calibration_label"],
191  )
192  sensorTransmission = cT.PrerequisiteInput(
193  name="transmission_sensor",
194  storageClass="TransmissionCurve",
195  doc="Transmission curve due to the sensor.",
196  dimensions=["instrument", "calibration_label", "detector"],
197  )
198  atmosphereTransmission = cT.PrerequisiteInput(
199  name="transmission_atmosphere",
200  storageClass="TransmissionCurve",
201  doc="Transmission curve due to the atmosphere.",
202  dimensions=["instrument"],
203  )
204  illumMaskedImage = cT.PrerequisiteInput(
205  name="illum",
206  doc="Input illumination correction.",
207  storageClass="MaskedImageF",
208  dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
209  )
210 
211  outputExposure = cT.Output(
212  name='postISRCCD',
213  doc="Output ISR processed exposure.",
214  storageClass="Exposure",
215  dimensions=["instrument", "exposure", "detector"],
216  )
217  preInterpExposure = cT.Output(
218  name='preInterpISRCCD',
219  doc="Output ISR processed exposure, with pixels left uninterpolated.",
220  storageClass="ExposureF",
221  dimensions=["instrument", "exposure", "detector"],
222  )
223  outputOssThumbnail = cT.Output(
224  name="OssThumb",
225  doc="Output Overscan-subtracted thumbnail image.",
226  storageClass="Thumbnail",
227  dimensions=["instrument", "exposure", "detector"],
228  )
229  outputFlattenedThumbnail = cT.Output(
230  name="FlattenedThumb",
231  doc="Output flat-corrected thumbnail image.",
232  storageClass="Thumbnail",
233  dimensions=["instrument", "exposure", "detector"],
234  )
235 
236  def __init__(self, *, config=None):
237  super().__init__(config=config)
238 
239  if config.doBias is not True:
240  self.prerequisiteInputs.discard("bias")
241  if config.doLinearize is not True:
242  self.prerequisiteInputs.discard("linearizer")
243  if config.doCrosstalk is not True:
244  self.inputs.discard("crosstalkSources")
245  self.prerequisiteInputs.discard("crosstalk")
246  if config.doBrighterFatter is not True:
247  self.prerequisiteInputs.discard("bfKernel")
248  self.prerequisiteInputs.discard("newBFKernel")
249  if config.doDefect is not True:
250  self.prerequisiteInputs.discard("defects")
251  if config.doDark is not True:
252  self.prerequisiteInputs.discard("dark")
253  if config.doFlat is not True:
254  self.prerequisiteInputs.discard("flat")
255  if config.doAttachTransmissionCurve is not True:
256  self.prerequisiteInputs.discard("opticsTransmission")
257  self.prerequisiteInputs.discard("filterTransmission")
258  self.prerequisiteInputs.discard("sensorTransmission")
259  self.prerequisiteInputs.discard("atmosphereTransmission")
260  if config.doUseOpticsTransmission is not True:
261  self.prerequisiteInputs.discard("opticsTransmission")
262  if config.doUseFilterTransmission is not True:
263  self.prerequisiteInputs.discard("filterTransmission")
264  if config.doUseSensorTransmission is not True:
265  self.prerequisiteInputs.discard("sensorTransmission")
266  if config.doUseAtmosphereTransmission is not True:
267  self.prerequisiteInputs.discard("atmosphereTransmission")
268  if config.doIlluminationCorrection is not True:
269  self.prerequisiteInputs.discard("illumMaskedImage")
270 
271  if config.doWrite is not True:
272  self.outputs.discard("outputExposure")
273  self.outputs.discard("preInterpExposure")
274  self.outputs.discard("outputFlattenedThumbnail")
275  self.outputs.discard("outputOssThumbnail")
276  if config.doSaveInterpPixels is not True:
277  self.outputs.discard("preInterpExposure")
278  if config.qa.doThumbnailOss is not True:
279  self.outputs.discard("outputOssThumbnail")
280  if config.qa.doThumbnailFlattened is not True:
281  self.outputs.discard("outputFlattenedThumbnail")
282 
283 
284 class IsrTaskConfig(pipeBase.PipelineTaskConfig,
285  pipelineConnections=IsrTaskConnections):
286  """Configuration parameters for IsrTask.
287 
288  Items are grouped in the order in which they are executed by the task.
289  """
290  datasetType = pexConfig.Field(
291  dtype=str,
292  doc="Dataset type for input data; users will typically leave this alone, "
293  "but camera-specific ISR tasks will override it",
294  default="raw",
295  )
296 
297  fallbackFilterName = pexConfig.Field(
298  dtype=str,
299  doc="Fallback default filter name for calibrations.",
300  optional=True
301  )
302  useFallbackDate = pexConfig.Field(
303  dtype=bool,
304  doc="Pass observation date when using fallback filter.",
305  default=False,
306  )
307  expectWcs = pexConfig.Field(
308  dtype=bool,
309  default=True,
310  doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
311  )
312  fwhm = pexConfig.Field(
313  dtype=float,
314  doc="FWHM of PSF in arcseconds.",
315  default=1.0,
316  )
317  qa = pexConfig.ConfigField(
318  dtype=isrQa.IsrQaConfig,
319  doc="QA related configuration options.",
320  )
321 
322  # Image conversion configuration
323  doConvertIntToFloat = pexConfig.Field(
324  dtype=bool,
325  doc="Convert integer raw images to floating point values?",
326  default=True,
327  )
328 
329  # Saturated pixel handling.
330  doSaturation = pexConfig.Field(
331  dtype=bool,
332  doc="Mask saturated pixels? NB: this is totally independent of the"
333  " interpolation option - this is ONLY setting the bits in the mask."
334  " To have them interpolated make sure doSaturationInterpolation=True",
335  default=True,
336  )
337  saturatedMaskName = pexConfig.Field(
338  dtype=str,
339  doc="Name of mask plane to use in saturation detection and interpolation",
340  default="SAT",
341  )
342  saturation = pexConfig.Field(
343  dtype=float,
344  doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
345  default=float("NaN"),
346  )
347  growSaturationFootprintSize = pexConfig.Field(
348  dtype=int,
349  doc="Number of pixels by which to grow the saturation footprints",
350  default=1,
351  )
352 
353  # Suspect pixel handling.
354  doSuspect = pexConfig.Field(
355  dtype=bool,
356  doc="Mask suspect pixels?",
357  default=False,
358  )
359  suspectMaskName = pexConfig.Field(
360  dtype=str,
361  doc="Name of mask plane to use for suspect pixels",
362  default="SUSPECT",
363  )
364  numEdgeSuspect = pexConfig.Field(
365  dtype=int,
366  doc="Number of edge pixels to be flagged as untrustworthy.",
367  default=0,
368  )
369 
370  # Initial masking options.
371  doSetBadRegions = pexConfig.Field(
372  dtype=bool,
373  doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
374  default=True,
375  )
376  badStatistic = pexConfig.ChoiceField(
377  dtype=str,
378  doc="How to estimate the average value for BAD regions.",
379  default='MEANCLIP',
380  allowed={
381  "MEANCLIP": "Correct using the (clipped) mean of good data",
382  "MEDIAN": "Correct using the median of the good data",
383  },
384  )
385 
386  # Overscan subtraction configuration.
387  doOverscan = pexConfig.Field(
388  dtype=bool,
389  doc="Do overscan subtraction?",
390  default=True,
391  )
392  overscan = pexConfig.ConfigurableField(
393  target=OverscanCorrectionTask,
394  doc="Overscan subtraction task for image segments.",
395  )
396 
397  overscanFitType = pexConfig.ChoiceField(
398  dtype=str,
399  doc="The method for fitting the overscan bias level.",
400  default='MEDIAN',
401  allowed={
402  "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
403  "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
404  "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
405  "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
406  "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
407  "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
408  "MEAN": "Correct using the mean of the overscan region",
409  "MEANCLIP": "Correct using a clipped mean of the overscan region",
410  "MEDIAN": "Correct using the median of the overscan region",
411  "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
412  },
413  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
414  " This option will no longer be used, and will be removed after v20.")
415  )
416  overscanOrder = pexConfig.Field(
417  dtype=int,
418  doc=("Order of polynomial or to fit if overscan fit type is a polynomial, "
419  "or number of spline knots if overscan fit type is a spline."),
420  default=1,
421  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
422  " This option will no longer be used, and will be removed after v20.")
423  )
424  overscanNumSigmaClip = pexConfig.Field(
425  dtype=float,
426  doc="Rejection threshold (sigma) for collapsing overscan before fit",
427  default=3.0,
428  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
429  " This option will no longer be used, and will be removed after v20.")
430  )
431  overscanIsInt = pexConfig.Field(
432  dtype=bool,
433  doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
434  " and overscan.FitType=MEDIAN_PER_ROW.",
435  default=True,
436  deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
437  " This option will no longer be used, and will be removed after v20.")
438  )
439  # These options do not get deprecated, as they define how we slice up the image data.
440  overscanNumLeadingColumnsToSkip = pexConfig.Field(
441  dtype=int,
442  doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
443  default=0,
444  )
445  overscanNumTrailingColumnsToSkip = pexConfig.Field(
446  dtype=int,
447  doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
448  default=0,
449  )
450  overscanMaxDev = pexConfig.Field(
451  dtype=float,
452  doc="Maximum deviation from the median for overscan",
453  default=1000.0, check=lambda x: x > 0
454  )
455  overscanBiasJump = pexConfig.Field(
456  dtype=bool,
457  doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
458  default=False,
459  )
460  overscanBiasJumpKeyword = pexConfig.Field(
461  dtype=str,
462  doc="Header keyword containing information about devices.",
463  default="NO_SUCH_KEY",
464  )
465  overscanBiasJumpDevices = pexConfig.ListField(
466  dtype=str,
467  doc="List of devices that need piecewise overscan correction.",
468  default=(),
469  )
470  overscanBiasJumpLocation = pexConfig.Field(
471  dtype=int,
472  doc="Location of bias jump along y-axis.",
473  default=0,
474  )
475 
476  # Amplifier to CCD assembly configuration
477  doAssembleCcd = pexConfig.Field(
478  dtype=bool,
479  default=True,
480  doc="Assemble amp-level exposures into a ccd-level exposure?"
481  )
482  assembleCcd = pexConfig.ConfigurableField(
483  target=AssembleCcdTask,
484  doc="CCD assembly task",
485  )
486 
487  # General calibration configuration.
488  doAssembleIsrExposures = pexConfig.Field(
489  dtype=bool,
490  default=False,
491  doc="Assemble amp-level calibration exposures into ccd-level exposure?"
492  )
493  doTrimToMatchCalib = pexConfig.Field(
494  dtype=bool,
495  default=False,
496  doc="Trim raw data to match calibration bounding boxes?"
497  )
498 
499  # Bias subtraction.
500  doBias = pexConfig.Field(
501  dtype=bool,
502  doc="Apply bias frame correction?",
503  default=True,
504  )
505  biasDataProductName = pexConfig.Field(
506  dtype=str,
507  doc="Name of the bias data product",
508  default="bias",
509  )
510 
511  # Variance construction
512  doVariance = pexConfig.Field(
513  dtype=bool,
514  doc="Calculate variance?",
515  default=True
516  )
517  gain = pexConfig.Field(
518  dtype=float,
519  doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
520  default=float("NaN"),
521  )
522  readNoise = pexConfig.Field(
523  dtype=float,
524  doc="The read noise to use if no Detector is present in the Exposure",
525  default=0.0,
526  )
527  doEmpiricalReadNoise = pexConfig.Field(
528  dtype=bool,
529  default=False,
530  doc="Calculate empirical read noise instead of value from AmpInfo data?"
531  )
532 
533  # Linearization.
534  doLinearize = pexConfig.Field(
535  dtype=bool,
536  doc="Correct for nonlinearity of the detector's response?",
537  default=True,
538  )
539 
540  # Crosstalk.
541  doCrosstalk = pexConfig.Field(
542  dtype=bool,
543  doc="Apply intra-CCD crosstalk correction?",
544  default=False,
545  )
546  doCrosstalkBeforeAssemble = pexConfig.Field(
547  dtype=bool,
548  doc="Apply crosstalk correction before CCD assembly, and before trimming?",
549  default=False,
550  )
551  crosstalk = pexConfig.ConfigurableField(
552  target=CrosstalkTask,
553  doc="Intra-CCD crosstalk correction",
554  )
555 
556  # Masking options.
557  doDefect = pexConfig.Field(
558  dtype=bool,
559  doc="Apply correction for CCD defects, e.g. hot pixels?",
560  default=True,
561  )
562  doNanMasking = pexConfig.Field(
563  dtype=bool,
564  doc="Mask NAN pixels?",
565  default=True,
566  )
567  doWidenSaturationTrails = pexConfig.Field(
568  dtype=bool,
569  doc="Widen bleed trails based on their width?",
570  default=True
571  )
572 
573  # Brighter-Fatter correction.
574  doBrighterFatter = pexConfig.Field(
575  dtype=bool,
576  default=False,
577  doc="Apply the brighter fatter correction"
578  )
579  brighterFatterLevel = pexConfig.ChoiceField(
580  dtype=str,
581  default="DETECTOR",
582  doc="The level at which to correct for brighter-fatter.",
583  allowed={
584  "AMP": "Every amplifier treated separately.",
585  "DETECTOR": "One kernel per detector",
586  }
587  )
588  brighterFatterMaxIter = pexConfig.Field(
589  dtype=int,
590  default=10,
591  doc="Maximum number of iterations for the brighter fatter correction"
592  )
593  brighterFatterThreshold = pexConfig.Field(
594  dtype=float,
595  default=1000,
596  doc="Threshold used to stop iterating the brighter fatter correction. It is the "
597  " absolute value of the difference between the current corrected image and the one"
598  " from the previous iteration summed over all the pixels."
599  )
600  brighterFatterApplyGain = pexConfig.Field(
601  dtype=bool,
602  default=True,
603  doc="Should the gain be applied when applying the brighter fatter correction?"
604  )
605  brighterFatterMaskGrowSize = pexConfig.Field(
606  dtype=int,
607  default=0,
608  doc="Number of pixels to grow the masks listed in config.maskListToInterpolate "
609  " when brighter-fatter correction is applied."
610  )
611 
612  # Dark subtraction.
613  doDark = pexConfig.Field(
614  dtype=bool,
615  doc="Apply dark frame correction?",
616  default=True,
617  )
618  darkDataProductName = pexConfig.Field(
619  dtype=str,
620  doc="Name of the dark data product",
621  default="dark",
622  )
623 
624  # Camera-specific stray light removal.
625  doStrayLight = pexConfig.Field(
626  dtype=bool,
627  doc="Subtract stray light in the y-band (due to encoder LEDs)?",
628  default=False,
629  )
630  strayLight = pexConfig.ConfigurableField(
631  target=StrayLightTask,
632  doc="y-band stray light correction"
633  )
634 
635  # Flat correction.
636  doFlat = pexConfig.Field(
637  dtype=bool,
638  doc="Apply flat field correction?",
639  default=True,
640  )
641  flatDataProductName = pexConfig.Field(
642  dtype=str,
643  doc="Name of the flat data product",
644  default="flat",
645  )
646  flatScalingType = pexConfig.ChoiceField(
647  dtype=str,
648  doc="The method for scaling the flat on the fly.",
649  default='USER',
650  allowed={
651  "USER": "Scale by flatUserScale",
652  "MEAN": "Scale by the inverse of the mean",
653  "MEDIAN": "Scale by the inverse of the median",
654  },
655  )
656  flatUserScale = pexConfig.Field(
657  dtype=float,
658  doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
659  default=1.0,
660  )
661  doTweakFlat = pexConfig.Field(
662  dtype=bool,
663  doc="Tweak flats to match observed amplifier ratios?",
664  default=False
665  )
666 
667  # Amplifier normalization based on gains instead of using flats configuration.
668  doApplyGains = pexConfig.Field(
669  dtype=bool,
670  doc="Correct the amplifiers for their gains instead of applying flat correction",
671  default=False,
672  )
673  normalizeGains = pexConfig.Field(
674  dtype=bool,
675  doc="Normalize all the amplifiers in each CCD to have the same median value.",
676  default=False,
677  )
678 
679  # Fringe correction.
680  doFringe = pexConfig.Field(
681  dtype=bool,
682  doc="Apply fringe correction?",
683  default=True,
684  )
685  fringe = pexConfig.ConfigurableField(
686  target=FringeTask,
687  doc="Fringe subtraction task",
688  )
689  fringeAfterFlat = pexConfig.Field(
690  dtype=bool,
691  doc="Do fringe subtraction after flat-fielding?",
692  default=True,
693  )
694 
695  # Initial CCD-level background statistics options.
696  doMeasureBackground = pexConfig.Field(
697  dtype=bool,
698  doc="Measure the background level on the reduced image?",
699  default=False,
700  )
701 
702  # Camera-specific masking configuration.
703  doCameraSpecificMasking = pexConfig.Field(
704  dtype=bool,
705  doc="Mask camera-specific bad regions?",
706  default=False,
707  )
708  masking = pexConfig.ConfigurableField(
709  target=MaskingTask,
710  doc="Masking task."
711  )
712 
713  # Interpolation options.
714 
715  doInterpolate = pexConfig.Field(
716  dtype=bool,
717  doc="Interpolate masked pixels?",
718  default=True,
719  )
720  doSaturationInterpolation = pexConfig.Field(
721  dtype=bool,
722  doc="Perform interpolation over pixels masked as saturated?"
723  " NB: This is independent of doSaturation; if that is False this plane"
724  " will likely be blank, resulting in a no-op here.",
725  default=True,
726  )
727  doNanInterpolation = pexConfig.Field(
728  dtype=bool,
729  doc="Perform interpolation over pixels masked as NaN?"
730  " NB: This is independent of doNanMasking; if that is False this plane"
731  " will likely be blank, resulting in a no-op here.",
732  default=True,
733  )
734  doNanInterpAfterFlat = pexConfig.Field(
735  dtype=bool,
736  doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
737  "also have to interpolate them before flat-fielding."),
738  default=False,
739  )
740  maskListToInterpolate = pexConfig.ListField(
741  dtype=str,
742  doc="List of mask planes that should be interpolated.",
743  default=['SAT', 'BAD', 'UNMASKEDNAN'],
744  )
745  doSaveInterpPixels = pexConfig.Field(
746  dtype=bool,
747  doc="Save a copy of the pre-interpolated pixel values?",
748  default=False,
749  )
750 
751  # Default photometric calibration options.
752  fluxMag0T1 = pexConfig.DictField(
753  keytype=str,
754  itemtype=float,
755  doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
756  default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
757  ))
758  )
759  defaultFluxMag0T1 = pexConfig.Field(
760  dtype=float,
761  doc="Default value for fluxMag0T1 (for an unrecognized filter).",
762  default=pow(10.0, 0.4*28.0)
763  )
764 
765  # Vignette correction configuration.
766  doVignette = pexConfig.Field(
767  dtype=bool,
768  doc="Apply vignetting parameters?",
769  default=False,
770  )
771  vignette = pexConfig.ConfigurableField(
772  target=VignetteTask,
773  doc="Vignetting task.",
774  )
775 
776  # Transmission curve configuration.
777  doAttachTransmissionCurve = pexConfig.Field(
778  dtype=bool,
779  default=False,
780  doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
781  )
782  doUseOpticsTransmission = pexConfig.Field(
783  dtype=bool,
784  default=True,
785  doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
786  )
787  doUseFilterTransmission = pexConfig.Field(
788  dtype=bool,
789  default=True,
790  doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
791  )
792  doUseSensorTransmission = pexConfig.Field(
793  dtype=bool,
794  default=True,
795  doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
796  )
797  doUseAtmosphereTransmission = pexConfig.Field(
798  dtype=bool,
799  default=True,
800  doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
801  )
802 
803  # Illumination correction.
804  doIlluminationCorrection = pexConfig.Field(
805  dtype=bool,
806  default=False,
807  doc="Perform illumination correction?"
808  )
809  illuminationCorrectionDataProductName = pexConfig.Field(
810  dtype=str,
811  doc="Name of the illumination correction data product.",
812  default="illumcor",
813  )
814  illumScale = pexConfig.Field(
815  dtype=float,
816  doc="Scale factor for the illumination correction.",
817  default=1.0,
818  )
819  illumFilters = pexConfig.ListField(
820  dtype=str,
821  default=[],
822  doc="Only perform illumination correction for these filters."
823  )
824 
825  # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
826  doWrite = pexConfig.Field(
827  dtype=bool,
828  doc="Persist postISRCCD?",
829  default=True,
830  )
831 
832  def validate(self):
833  super().validate()
834  if self.doFlat and self.doApplyGains:
835  raise ValueError("You may not specify both doFlat and doApplyGains")
836  if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate:
837  self.config.maskListToInterpolate.append("SAT")
838  if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
839  self.config.maskListToInterpolate.append("UNMASKEDNAN")
840 
841 
842 class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
843  """Apply common instrument signature correction algorithms to a raw frame.
844 
845  The process for correcting imaging data is very similar from
846  camera to camera. This task provides a vanilla implementation of
847  doing these corrections, including the ability to turn certain
848  corrections off if they are not needed. The inputs to the primary
849  method, `run()`, are a raw exposure to be corrected and the
850  calibration data products. The raw input is a single chip sized
851  mosaic of all amps including overscans and other non-science
852  pixels. The method `runDataRef()` identifies and defines the
853  calibration data products, and is intended for use by a
854  `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
855  `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
856  subclassed for different camera, although the most camera specific
857  methods have been split into subtasks that can be redirected
858  appropriately.
859 
860  The __init__ method sets up the subtasks for ISR processing, using
861  the defaults from `lsst.ip.isr`.
862 
863  Parameters
864  ----------
865  args : `list`
866  Positional arguments passed to the Task constructor. None used at this time.
867  kwargs : `dict`, optional
868  Keyword arguments passed on to the Task constructor. None used at this time.
869  """
870  ConfigClass = IsrTaskConfig
871  _DefaultName = "isr"
872 
873  def __init__(self, **kwargs):
874  super().__init__(**kwargs)
875  self.makeSubtask("assembleCcd")
876  self.makeSubtask("crosstalk")
877  self.makeSubtask("strayLight")
878  self.makeSubtask("fringe")
879  self.makeSubtask("masking")
880  self.makeSubtask("overscan")
881  self.makeSubtask("vignette")
882 
883  def runQuantum(self, butlerQC, inputRefs, outputRefs):
884  inputs = butlerQC.get(inputRefs)
885 
886  try:
887  inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
888  except Exception as e:
889  raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
890  (inputRefs, e))
891 
892  inputs['isGen3'] = True
893 
894  detector = inputs['ccdExposure'].getDetector()
895 
896  if self.config.doCrosstalk is True:
897  # Crosstalk sources need to be defined by the pipeline
898  # yaml if they exist.
899  if 'crosstalk' in inputs and inputs['crosstalk'] is not None:
900  if not isinstance(inputs['crosstalk'], CrosstalkCalib):
901  inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk'])
902  else:
903  coeffVector = (self.config.crosstalk.crosstalkValues
904  if self.config.crosstalk.useConfigCoefficients else None)
905  crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
906  inputs['crosstalk'] = crosstalkCalib
907  if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0:
908  if 'crosstalkSources' not in inputs:
909  self.log.warn("No crosstalkSources found for chip with interChip terms!")
910 
911  if self.doLinearize(detector) is True:
912  if 'linearizer' in inputs and isinstance(inputs['linearizer'], dict):
913  linearizer = linearize.Linearizer(detector=detector, log=self.log)
914  linearizer.fromYaml(inputs['linearizer'])
915  else:
916  linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), detector=detector,
917  log=self.log)
918  inputs['linearizer'] = linearizer
919 
920  if self.config.doDefect is True:
921  if "defects" in inputs and inputs['defects'] is not None:
922  # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
923  # masking expects a list of defects defined by their bounding box
924  if not isinstance(inputs["defects"], Defects):
925  inputs["defects"] = Defects.fromTable(inputs["defects"])
926 
927  # Load the correct style of brighter fatter kernel, and repack
928  # the information as a numpy array.
929  if self.config.doBrighterFatter:
930  brighterFatterKernel = inputs.pop('newBFKernel', None)
931  if brighterFatterKernel is None:
932  brighterFatterKernel = inputs.get('bfKernel', None)
933 
934  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
935  detId = detector.getId()
936  inputs['bfGains'] = brighterFatterKernel.gain
937  # If the kernel is not an ndarray, it's the cp_pipe version
938  # so extract the kernel for this detector, or raise an error
939  if self.config.brighterFatterLevel == 'DETECTOR':
940  if brighterFatterKernel.detectorKernel:
941  inputs['bfKernel'] = brighterFatterKernel.detectorKernel[detId]
942  elif brighterFatterKernel.detectorKernelFromAmpKernels:
943  inputs['bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
944  else:
945  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
946  else:
947  # TODO DM-15631 for implementing this
948  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
949 
950  if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
951  expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
952  inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
953  expId=expId,
954  assembler=self.assembleCcd
955  if self.config.doAssembleIsrExposures else None)
956  else:
957  inputs['fringes'] = pipeBase.Struct(fringes=None)
958 
959  if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
960  if 'strayLightData' not in inputs:
961  inputs['strayLightData'] = None
962 
963  outputs = self.run(**inputs)
964  butlerQC.put(outputs, outputRefs)
965 
966  def readIsrData(self, dataRef, rawExposure):
967  """!Retrieve necessary frames for instrument signature removal.
968 
969  Pre-fetching all required ISR data products limits the IO
970  required by the ISR. Any conflict between the calibration data
971  available and that needed for ISR is also detected prior to
972  doing processing, allowing it to fail quickly.
973 
974  Parameters
975  ----------
976  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
977  Butler reference of the detector data to be processed
978  rawExposure : `afw.image.Exposure`
979  The raw exposure that will later be corrected with the
980  retrieved calibration data; should not be modified in this
981  method.
982 
983  Returns
984  -------
985  result : `lsst.pipe.base.Struct`
986  Result struct with components (which may be `None`):
987  - ``bias``: bias calibration frame (`afw.image.Exposure`)
988  - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
989  - ``crosstalkSources``: list of possible crosstalk sources (`list`)
990  - ``dark``: dark calibration frame (`afw.image.Exposure`)
991  - ``flat``: flat calibration frame (`afw.image.Exposure`)
992  - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
993  - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
994  - ``fringes``: `lsst.pipe.base.Struct` with components:
995  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
996  - ``seed``: random seed derived from the ccdExposureId for random
997  number generator (`uint32`).
998  - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
999  A ``TransmissionCurve`` that represents the throughput of the optics,
1000  to be evaluated in focal-plane coordinates.
1001  - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1002  A ``TransmissionCurve`` that represents the throughput of the filter
1003  itself, to be evaluated in focal-plane coordinates.
1004  - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1005  A ``TransmissionCurve`` that represents the throughput of the sensor
1006  itself, to be evaluated in post-assembly trimmed detector coordinates.
1007  - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1008  A ``TransmissionCurve`` that represents the throughput of the
1009  atmosphere, assumed to be spatially constant.
1010  - ``strayLightData`` : `object`
1011  An opaque object containing calibration information for
1012  stray-light correction. If `None`, no correction will be
1013  performed.
1014  - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1015 
1016  Raises
1017  ------
1018  NotImplementedError :
1019  Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1020  """
1021  try:
1022  dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1023  dateObs = dateObs.toPython().isoformat()
1024  except RuntimeError:
1025  self.log.warn("Unable to identify dateObs for rawExposure.")
1026  dateObs = None
1027 
1028  ccd = rawExposure.getDetector()
1029  filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
1030  rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
1031  biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
1032  if self.config.doBias else None)
1033  # immediate=True required for functors and linearizers are functors; see ticket DM-6515
1034  linearizer = (dataRef.get("linearizer", immediate=True)
1035  if self.doLinearize(ccd) else None)
1036  if linearizer is not None and not isinstance(linearizer, numpy.ndarray):
1037  linearizer.log = self.log
1038  if isinstance(linearizer, numpy.ndarray):
1039  linearizer = linearize.Linearizer(table=linearizer, detector=ccd)
1040 
1041  crosstalkCalib = None
1042  if self.config.doCrosstalk:
1043  try:
1044  crosstalkCalib = dataRef.get("crosstalk", immediate=True)
1045  except NoResults:
1046  coeffVector = (self.config.crosstalk.crosstalkValues
1047  if self.config.crosstalk.useConfigCoefficients else None)
1048  crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1049  crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1050  if self.config.doCrosstalk else None)
1051 
1052  darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
1053  if self.config.doDark else None)
1054  flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName,
1055  dateObs=dateObs)
1056  if self.config.doFlat else None)
1057 
1058  brighterFatterKernel = None
1059  brighterFatterGains = None
1060  if self.config.doBrighterFatter is True:
1061  try:
1062  # Use the new-style cp_pipe version of the kernel if it exists
1063  # If using a new-style kernel, always use the self-consistent
1064  # gains, i.e. the ones inside the kernel object itself
1065  brighterFatterKernel = dataRef.get("brighterFatterKernel")
1066  brighterFatterGains = brighterFatterKernel.gain
1067  self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded")
1068  except NoResults:
1069  try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
1070  brighterFatterKernel = dataRef.get("bfKernel")
1071  self.log.info("Old style bright-fatter kernel (np.array) loaded")
1072  except NoResults:
1073  brighterFatterKernel = None
1074  if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1075  # If the kernel is not an ndarray, it's the cp_pipe version
1076  # so extract the kernel for this detector, or raise an error
1077  if self.config.brighterFatterLevel == 'DETECTOR':
1078  if brighterFatterKernel.detectorKernel:
1079  brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
1080  elif brighterFatterKernel.detectorKernelFromAmpKernels:
1081  brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
1082  else:
1083  raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1084  else:
1085  # TODO DM-15631 for implementing this
1086  raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1087 
1088  defectList = (dataRef.get("defects")
1089  if self.config.doDefect else None)
1090  fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
1091  if self.config.doAssembleIsrExposures else None)
1092  if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1093  else pipeBase.Struct(fringes=None))
1094 
1095  if self.config.doAttachTransmissionCurve:
1096  opticsTransmission = (dataRef.get("transmission_optics")
1097  if self.config.doUseOpticsTransmission else None)
1098  filterTransmission = (dataRef.get("transmission_filter")
1099  if self.config.doUseFilterTransmission else None)
1100  sensorTransmission = (dataRef.get("transmission_sensor")
1101  if self.config.doUseSensorTransmission else None)
1102  atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1103  if self.config.doUseAtmosphereTransmission else None)
1104  else:
1105  opticsTransmission = None
1106  filterTransmission = None
1107  sensorTransmission = None
1108  atmosphereTransmission = None
1109 
1110  if self.config.doStrayLight:
1111  strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1112  else:
1113  strayLightData = None
1114 
1115  illumMaskedImage = (self.getIsrExposure(dataRef,
1116  self.config.illuminationCorrectionDataProductName).getMaskedImage()
1117  if (self.config.doIlluminationCorrection
1118  and filterName in self.config.illumFilters)
1119  else None)
1120 
1121  # Struct should include only kwargs to run()
1122  return pipeBase.Struct(bias=biasExposure,
1123  linearizer=linearizer,
1124  crosstalk=crosstalkCalib,
1125  crosstalkSources=crosstalkSources,
1126  dark=darkExposure,
1127  flat=flatExposure,
1128  bfKernel=brighterFatterKernel,
1129  bfGains=brighterFatterGains,
1130  defects=defectList,
1131  fringes=fringeStruct,
1132  opticsTransmission=opticsTransmission,
1133  filterTransmission=filterTransmission,
1134  sensorTransmission=sensorTransmission,
1135  atmosphereTransmission=atmosphereTransmission,
1136  strayLightData=strayLightData,
1137  illumMaskedImage=illumMaskedImage
1138  )
1139 
1140  @pipeBase.timeMethod
1141  def run(self, ccdExposure, camera=None, bias=None, linearizer=None,
1142  crosstalk=None, crosstalkSources=None,
1143  dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1144  fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1145  sensorTransmission=None, atmosphereTransmission=None,
1146  detectorNum=None, strayLightData=None, illumMaskedImage=None,
1147  isGen3=False,
1148  ):
1149  """!Perform instrument signature removal on an exposure.
1150 
1151  Steps included in the ISR processing, in order performed, are:
1152  - saturation and suspect pixel masking
1153  - overscan subtraction
1154  - CCD assembly of individual amplifiers
1155  - bias subtraction
1156  - variance image construction
1157  - linearization of non-linear response
1158  - crosstalk masking
1159  - brighter-fatter correction
1160  - dark subtraction
1161  - fringe correction
1162  - stray light subtraction
1163  - flat correction
1164  - masking of known defects and camera specific features
1165  - vignette calculation
1166  - appending transmission curve and distortion model
1167 
1168  Parameters
1169  ----------
1170  ccdExposure : `lsst.afw.image.Exposure`
1171  The raw exposure that is to be run through ISR. The
1172  exposure is modified by this method.
1173  camera : `lsst.afw.cameraGeom.Camera`, optional
1174  The camera geometry for this exposure. Required if ``isGen3`` is
1175  `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1176  ``flat`` does not have an associated detector.
1177  bias : `lsst.afw.image.Exposure`, optional
1178  Bias calibration frame.
1179  linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1180  Functor for linearization.
1181  crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1182  Calibration for crosstalk.
1183  crosstalkSources : `list`, optional
1184  List of possible crosstalk sources.
1185  dark : `lsst.afw.image.Exposure`, optional
1186  Dark calibration frame.
1187  flat : `lsst.afw.image.Exposure`, optional
1188  Flat calibration frame.
1189  bfKernel : `numpy.ndarray`, optional
1190  Brighter-fatter kernel.
1191  bfGains : `dict` of `float`, optional
1192  Gains used to override the detector's nominal gains for the
1193  brighter-fatter correction. A dict keyed by amplifier name for
1194  the detector in question.
1195  defects : `lsst.meas.algorithms.Defects`, optional
1196  List of defects.
1197  fringes : `lsst.pipe.base.Struct`, optional
1198  Struct containing the fringe correction data, with
1199  elements:
1200  - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1201  - ``seed``: random seed derived from the ccdExposureId for random
1202  number generator (`uint32`)
1203  opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1204  A ``TransmissionCurve`` that represents the throughput of the optics,
1205  to be evaluated in focal-plane coordinates.
1206  filterTransmission : `lsst.afw.image.TransmissionCurve`
1207  A ``TransmissionCurve`` that represents the throughput of the filter
1208  itself, to be evaluated in focal-plane coordinates.
1209  sensorTransmission : `lsst.afw.image.TransmissionCurve`
1210  A ``TransmissionCurve`` that represents the throughput of the sensor
1211  itself, to be evaluated in post-assembly trimmed detector coordinates.
1212  atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1213  A ``TransmissionCurve`` that represents the throughput of the
1214  atmosphere, assumed to be spatially constant.
1215  detectorNum : `int`, optional
1216  The integer number for the detector to process.
1217  isGen3 : bool, optional
1218  Flag this call to run() as using the Gen3 butler environment.
1219  strayLightData : `object`, optional
1220  Opaque object containing calibration information for stray-light
1221  correction. If `None`, no correction will be performed.
1222  illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1223  Illumination correction image.
1224 
1225  Returns
1226  -------
1227  result : `lsst.pipe.base.Struct`
1228  Result struct with component:
1229  - ``exposure`` : `afw.image.Exposure`
1230  The fully ISR corrected exposure.
1231  - ``outputExposure`` : `afw.image.Exposure`
1232  An alias for `exposure`
1233  - ``ossThumb`` : `numpy.ndarray`
1234  Thumbnail image of the exposure after overscan subtraction.
1235  - ``flattenedThumb`` : `numpy.ndarray`
1236  Thumbnail image of the exposure after flat-field correction.
1237 
1238  Raises
1239  ------
1240  RuntimeError
1241  Raised if a configuration option is set to True, but the
1242  required calibration data has not been specified.
1243 
1244  Notes
1245  -----
1246  The current processed exposure can be viewed by setting the
1247  appropriate lsstDebug entries in the `debug.display`
1248  dictionary. The names of these entries correspond to some of
1249  the IsrTaskConfig Boolean options, with the value denoting the
1250  frame to use. The exposure is shown inside the matching
1251  option check and after the processing of that step has
1252  finished. The steps with debug points are:
1253 
1254  doAssembleCcd
1255  doBias
1256  doCrosstalk
1257  doBrighterFatter
1258  doDark
1259  doFringe
1260  doStrayLight
1261  doFlat
1262 
1263  In addition, setting the "postISRCCD" entry displays the
1264  exposure after all ISR processing has finished.
1265 
1266  """
1267 
1268  if isGen3 is True:
1269  # Gen3 currently cannot automatically do configuration overrides.
1270  # DM-15257 looks to discuss this issue.
1271  # Configure input exposures;
1272  if detectorNum is None:
1273  raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1274 
1275  ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1276  bias = self.ensureExposure(bias, camera, detectorNum)
1277  dark = self.ensureExposure(dark, camera, detectorNum)
1278  flat = self.ensureExposure(flat, camera, detectorNum)
1279  else:
1280  if isinstance(ccdExposure, ButlerDataRef):
1281  return self.runDataRef(ccdExposure)
1282 
1283  ccd = ccdExposure.getDetector()
1284  filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1285 
1286  if not ccd:
1287  assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1288  ccd = [FakeAmp(ccdExposure, self.config)]
1289 
1290  # Validate Input
1291  if self.config.doBias and bias is None:
1292  raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1293  if self.doLinearize(ccd) and linearizer is None:
1294  raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1295  if self.config.doBrighterFatter and bfKernel is None:
1296  raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1297  if self.config.doDark and dark is None:
1298  raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1299  if self.config.doFlat and flat is None:
1300  raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1301  if self.config.doDefect and defects is None:
1302  raise RuntimeError("Must supply defects if config.doDefect=True.")
1303  if (self.config.doFringe and filterName in self.fringe.config.filters
1304  and fringes.fringes is None):
1305  # The `fringes` object needs to be a pipeBase.Struct, as
1306  # we use it as a `dict` for the parameters of
1307  # `FringeTask.run()`. The `fringes.fringes` `list` may
1308  # not be `None` if `doFringe=True`. Otherwise, raise.
1309  raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1310  if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters
1311  and illumMaskedImage is None):
1312  raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1313 
1314  # Begin ISR processing.
1315  if self.config.doConvertIntToFloat:
1316  self.log.info("Converting exposure to floating point values.")
1317  ccdExposure = self.convertIntToFloat(ccdExposure)
1318 
1319  # Amplifier level processing.
1320  overscans = []
1321  for amp in ccd:
1322  # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1323  if ccdExposure.getBBox().contains(amp.getBBox()):
1324  # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1325  badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1326 
1327  if self.config.doOverscan and not badAmp:
1328  # Overscan correction on amp-by-amp basis.
1329  overscanResults = self.overscanCorrection(ccdExposure, amp)
1330  self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1331  if overscanResults is not None and \
1332  self.config.qa is not None and self.config.qa.saveStats is True:
1333  if isinstance(overscanResults.overscanFit, float):
1334  qaMedian = overscanResults.overscanFit
1335  qaStdev = float("NaN")
1336  else:
1337  qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1338  afwMath.MEDIAN | afwMath.STDEVCLIP)
1339  qaMedian = qaStats.getValue(afwMath.MEDIAN)
1340  qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1341 
1342  self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1343  self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1344  self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1345  amp.getName(), qaMedian, qaStdev)
1346  ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1347  else:
1348  if badAmp:
1349  self.log.warn("Amplifier %s is bad.", amp.getName())
1350  overscanResults = None
1351 
1352  overscans.append(overscanResults if overscanResults is not None else None)
1353  else:
1354  self.log.info("Skipped OSCAN for %s.", amp.getName())
1355 
1356  if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1357  self.log.info("Applying crosstalk correction.")
1358  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1359  crosstalkSources=crosstalkSources)
1360  self.debugView(ccdExposure, "doCrosstalk")
1361 
1362  if self.config.doAssembleCcd:
1363  self.log.info("Assembling CCD from amplifiers.")
1364  ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1365 
1366  if self.config.expectWcs and not ccdExposure.getWcs():
1367  self.log.warn("No WCS found in input exposure.")
1368  self.debugView(ccdExposure, "doAssembleCcd")
1369 
1370  ossThumb = None
1371  if self.config.qa.doThumbnailOss:
1372  ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1373 
1374  if self.config.doBias:
1375  self.log.info("Applying bias correction.")
1376  isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1377  trimToFit=self.config.doTrimToMatchCalib)
1378  self.debugView(ccdExposure, "doBias")
1379 
1380  if self.config.doVariance:
1381  for amp, overscanResults in zip(ccd, overscans):
1382  if ccdExposure.getBBox().contains(amp.getBBox()):
1383  self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1384  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1385  if overscanResults is not None:
1386  self.updateVariance(ampExposure, amp,
1387  overscanImage=overscanResults.overscanImage)
1388  else:
1389  self.updateVariance(ampExposure, amp,
1390  overscanImage=None)
1391  if self.config.qa is not None and self.config.qa.saveStats is True:
1392  qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1393  afwMath.MEDIAN | afwMath.STDEVCLIP)
1394  self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1395  qaStats.getValue(afwMath.MEDIAN))
1396  self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1397  qaStats.getValue(afwMath.STDEVCLIP))
1398  self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1399  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1400  qaStats.getValue(afwMath.STDEVCLIP))
1401 
1402  if self.doLinearize(ccd):
1403  self.log.info("Applying linearizer.")
1404  linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1405  detector=ccd, log=self.log)
1406 
1407  if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1408  self.log.info("Applying crosstalk correction.")
1409  self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1410  crosstalkSources=crosstalkSources, isTrimmed=True)
1411  self.debugView(ccdExposure, "doCrosstalk")
1412 
1413  # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1414  # anything else the camera needs. Saturated and suspect pixels have already been masked.
1415  if self.config.doDefect:
1416  self.log.info("Masking defects.")
1417  self.maskDefect(ccdExposure, defects)
1418 
1419  if self.config.numEdgeSuspect > 0:
1420  self.log.info("Masking edges as SUSPECT.")
1421  self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1422  maskPlane="SUSPECT")
1423 
1424  if self.config.doNanMasking:
1425  self.log.info("Masking NAN value pixels.")
1426  self.maskNan(ccdExposure)
1427 
1428  if self.config.doWidenSaturationTrails:
1429  self.log.info("Widening saturation trails.")
1430  isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1431 
1432  if self.config.doCameraSpecificMasking:
1433  self.log.info("Masking regions for camera specific reasons.")
1434  self.masking.run(ccdExposure)
1435 
1436  if self.config.doBrighterFatter:
1437  # We need to apply flats and darks before we can interpolate, and we
1438  # need to interpolate before we do B-F, but we do B-F without the
1439  # flats and darks applied so we can work in units of electrons or holes.
1440  # This context manager applies and then removes the darks and flats.
1441  #
1442  # We also do not want to interpolate values here, so operate on temporary
1443  # images so we can apply only the BF-correction and roll back the
1444  # interpolation.
1445  interpExp = ccdExposure.clone()
1446  with self.flatContext(interpExp, flat, dark):
1447  isrFunctions.interpolateFromMask(
1448  maskedImage=interpExp.getMaskedImage(),
1449  fwhm=self.config.fwhm,
1450  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1451  maskNameList=self.config.maskListToInterpolate
1452  )
1453  bfExp = interpExp.clone()
1454 
1455  self.log.info("Applying brighter fatter correction using kernel type %s / gains %s.",
1456  type(bfKernel), type(bfGains))
1457  bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1458  self.config.brighterFatterMaxIter,
1459  self.config.brighterFatterThreshold,
1460  self.config.brighterFatterApplyGain,
1461  bfGains)
1462  if bfResults[1] == self.config.brighterFatterMaxIter:
1463  self.log.warn("Brighter fatter correction did not converge, final difference %f.",
1464  bfResults[0])
1465  else:
1466  self.log.info("Finished brighter fatter correction in %d iterations.",
1467  bfResults[1])
1468  image = ccdExposure.getMaskedImage().getImage()
1469  bfCorr = bfExp.getMaskedImage().getImage()
1470  bfCorr -= interpExp.getMaskedImage().getImage()
1471  image += bfCorr
1472 
1473  # Applying the brighter-fatter correction applies a
1474  # convolution to the science image. At the edges this
1475  # convolution may not have sufficient valid pixels to
1476  # produce a valid correction. Mark pixels within the size
1477  # of the brighter-fatter kernel as EDGE to warn of this
1478  # fact.
1479  self.log.info("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1480  self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1481  maskPlane="EDGE")
1482 
1483  if self.config.brighterFatterMaskGrowSize > 0:
1484  self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1485  for maskPlane in self.config.maskListToInterpolate:
1486  isrFunctions.growMasks(ccdExposure.getMask(),
1487  radius=self.config.brighterFatterMaskGrowSize,
1488  maskNameList=maskPlane,
1489  maskValue=maskPlane)
1490 
1491  self.debugView(ccdExposure, "doBrighterFatter")
1492 
1493  if self.config.doDark:
1494  self.log.info("Applying dark correction.")
1495  self.darkCorrection(ccdExposure, dark)
1496  self.debugView(ccdExposure, "doDark")
1497 
1498  if self.config.doFringe and not self.config.fringeAfterFlat:
1499  self.log.info("Applying fringe correction before flat.")
1500  self.fringe.run(ccdExposure, **fringes.getDict())
1501  self.debugView(ccdExposure, "doFringe")
1502 
1503  if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1504  self.log.info("Checking strayLight correction.")
1505  self.strayLight.run(ccdExposure, strayLightData)
1506  self.debugView(ccdExposure, "doStrayLight")
1507 
1508  if self.config.doFlat:
1509  self.log.info("Applying flat correction.")
1510  self.flatCorrection(ccdExposure, flat)
1511  self.debugView(ccdExposure, "doFlat")
1512 
1513  if self.config.doApplyGains:
1514  self.log.info("Applying gain correction instead of flat.")
1515  isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1517  if self.config.doFringe and self.config.fringeAfterFlat:
1518  self.log.info("Applying fringe correction after flat.")
1519  self.fringe.run(ccdExposure, **fringes.getDict())
1520 
1521  if self.config.doVignette:
1522  self.log.info("Constructing Vignette polygon.")
1523  self.vignettePolygon = self.vignette.run(ccdExposure)
1524 
1525  if self.config.vignette.doWriteVignettePolygon:
1526  self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1527 
1528  if self.config.doAttachTransmissionCurve:
1529  self.log.info("Adding transmission curves.")
1530  isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1531  filterTransmission=filterTransmission,
1532  sensorTransmission=sensorTransmission,
1533  atmosphereTransmission=atmosphereTransmission)
1534 
1535  flattenedThumb = None
1536  if self.config.qa.doThumbnailFlattened:
1537  flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1538 
1539  if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1540  self.log.info("Performing illumination correction.")
1541  isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1542  illumMaskedImage, illumScale=self.config.illumScale,
1543  trimToFit=self.config.doTrimToMatchCalib)
1544 
1545  preInterpExp = None
1546  if self.config.doSaveInterpPixels:
1547  preInterpExp = ccdExposure.clone()
1548 
1549  # Reset and interpolate bad pixels.
1550  #
1551  # Large contiguous bad regions (which should have the BAD mask
1552  # bit set) should have their values set to the image median.
1553  # This group should include defects and bad amplifiers. As the
1554  # area covered by these defects are large, there's little
1555  # reason to expect that interpolation would provide a more
1556  # useful value.
1557  #
1558  # Smaller defects can be safely interpolated after the larger
1559  # regions have had their pixel values reset. This ensures
1560  # that the remaining defects adjacent to bad amplifiers (as an
1561  # example) do not attempt to interpolate extreme values.
1562  if self.config.doSetBadRegions:
1563  badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1564  if badPixelCount > 0:
1565  self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1566 
1567  if self.config.doInterpolate:
1568  self.log.info("Interpolating masked pixels.")
1569  isrFunctions.interpolateFromMask(
1570  maskedImage=ccdExposure.getMaskedImage(),
1571  fwhm=self.config.fwhm,
1572  growSaturatedFootprints=self.config.growSaturationFootprintSize,
1573  maskNameList=list(self.config.maskListToInterpolate)
1574  )
1575 
1576  self.roughZeroPoint(ccdExposure)
1577 
1578  if self.config.doMeasureBackground:
1579  self.log.info("Measuring background level.")
1580  self.measureBackground(ccdExposure, self.config.qa)
1581 
1582  if self.config.qa is not None and self.config.qa.saveStats is True:
1583  for amp in ccd:
1584  ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1585  qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1586  afwMath.MEDIAN | afwMath.STDEVCLIP)
1587  self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1588  qaStats.getValue(afwMath.MEDIAN))
1589  self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1590  qaStats.getValue(afwMath.STDEVCLIP))
1591  self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1592  amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1593  qaStats.getValue(afwMath.STDEVCLIP))
1594 
1595  self.debugView(ccdExposure, "postISRCCD")
1596 
1597  return pipeBase.Struct(
1598  exposure=ccdExposure,
1599  ossThumb=ossThumb,
1600  flattenedThumb=flattenedThumb,
1601 
1602  preInterpolatedExposure=preInterpExp,
1603  outputExposure=ccdExposure,
1604  outputOssThumbnail=ossThumb,
1605  outputFlattenedThumbnail=flattenedThumb,
1606  )
1607 
1608  @pipeBase.timeMethod
1609  def runDataRef(self, sensorRef):
1610  """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1611 
1612  This method contains the `CmdLineTask` interface to the ISR
1613  processing. All IO is handled here, freeing the `run()` method
1614  to manage only pixel-level calculations. The steps performed
1615  are:
1616  - Read in necessary detrending/isr/calibration data.
1617  - Process raw exposure in `run()`.
1618  - Persist the ISR-corrected exposure as "postISRCCD" if
1619  config.doWrite=True.
1620 
1621  Parameters
1622  ----------
1623  sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1624  DataRef of the detector data to be processed
1625 
1626  Returns
1627  -------
1628  result : `lsst.pipe.base.Struct`
1629  Result struct with component:
1630  - ``exposure`` : `afw.image.Exposure`
1631  The fully ISR corrected exposure.
1632 
1633  Raises
1634  ------
1635  RuntimeError
1636  Raised if a configuration option is set to True, but the
1637  required calibration data does not exist.
1638 
1639  """
1640  self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1641 
1642  ccdExposure = sensorRef.get(self.config.datasetType)
1643 
1644  camera = sensorRef.get("camera")
1645  isrData = self.readIsrData(sensorRef, ccdExposure)
1646 
1647  result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1648 
1649  if self.config.doWrite:
1650  sensorRef.put(result.exposure, "postISRCCD")
1651  if result.preInterpolatedExposure is not None:
1652  sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1653  if result.ossThumb is not None:
1654  isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1655  if result.flattenedThumb is not None:
1656  isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1657 
1658  return result
1659 
1660  def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1661  """!Retrieve a calibration dataset for removing instrument signature.
1662 
1663  Parameters
1664  ----------
1665 
1666  dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1667  DataRef of the detector data to find calibration datasets
1668  for.
1669  datasetType : `str`
1670  Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1671  dateObs : `str`, optional
1672  Date of the observation. Used to correct butler failures
1673  when using fallback filters.
1674  immediate : `Bool`
1675  If True, disable butler proxies to enable error handling
1676  within this routine.
1677 
1678  Returns
1679  -------
1680  exposure : `lsst.afw.image.Exposure`
1681  Requested calibration frame.
1682 
1683  Raises
1684  ------
1685  RuntimeError
1686  Raised if no matching calibration frame can be found.
1687  """
1688  try:
1689  exp = dataRef.get(datasetType, immediate=immediate)
1690  except Exception as exc1:
1691  if not self.config.fallbackFilterName:
1692  raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1693  try:
1694  if self.config.useFallbackDate and dateObs:
1695  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1696  dateObs=dateObs, immediate=immediate)
1697  else:
1698  exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1699  except Exception as exc2:
1700  raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1701  (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1702  self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1703 
1704  if self.config.doAssembleIsrExposures:
1705  exp = self.assembleCcd.assembleCcd(exp)
1706  return exp
1707 
1708  def ensureExposure(self, inputExp, camera, detectorNum):
1709  """Ensure that the data returned by Butler is a fully constructed exposure.
1710 
1711  ISR requires exposure-level image data for historical reasons, so if we did
1712  not recieve that from Butler, construct it from what we have, modifying the
1713  input in place.
1714 
1715  Parameters
1716  ----------
1717  inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1718  `lsst.afw.image.ImageF`
1719  The input data structure obtained from Butler.
1720  camera : `lsst.afw.cameraGeom.camera`
1721  The camera associated with the image. Used to find the appropriate
1722  detector.
1723  detectorNum : `int`
1724  The detector this exposure should match.
1725 
1726  Returns
1727  -------
1728  inputExp : `lsst.afw.image.Exposure`
1729  The re-constructed exposure, with appropriate detector parameters.
1730 
1731  Raises
1732  ------
1733  TypeError
1734  Raised if the input data cannot be used to construct an exposure.
1735  """
1736  if isinstance(inputExp, afwImage.DecoratedImageU):
1737  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1738  elif isinstance(inputExp, afwImage.ImageF):
1739  inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1740  elif isinstance(inputExp, afwImage.MaskedImageF):
1741  inputExp = afwImage.makeExposure(inputExp)
1742  elif isinstance(inputExp, afwImage.Exposure):
1743  pass
1744  elif inputExp is None:
1745  # Assume this will be caught by the setup if it is a problem.
1746  return inputExp
1747  else:
1748  raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1749  (type(inputExp), ))
1750 
1751  if inputExp.getDetector() is None:
1752  inputExp.setDetector(camera[detectorNum])
1753 
1754  return inputExp
1755 
1756  def convertIntToFloat(self, exposure):
1757  """Convert exposure image from uint16 to float.
1758 
1759  If the exposure does not need to be converted, the input is
1760  immediately returned. For exposures that are converted to use
1761  floating point pixels, the variance is set to unity and the
1762  mask to zero.
1763 
1764  Parameters
1765  ----------
1766  exposure : `lsst.afw.image.Exposure`
1767  The raw exposure to be converted.
1768 
1769  Returns
1770  -------
1771  newexposure : `lsst.afw.image.Exposure`
1772  The input ``exposure``, converted to floating point pixels.
1773 
1774  Raises
1775  ------
1776  RuntimeError
1777  Raised if the exposure type cannot be converted to float.
1778 
1779  """
1780  if isinstance(exposure, afwImage.ExposureF):
1781  # Nothing to be done
1782  self.log.debug("Exposure already of type float.")
1783  return exposure
1784  if not hasattr(exposure, "convertF"):
1785  raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1786 
1787  newexposure = exposure.convertF()
1788  newexposure.variance[:] = 1
1789  newexposure.mask[:] = 0x0
1790 
1791  return newexposure
1792 
1793  def maskAmplifier(self, ccdExposure, amp, defects):
1794  """Identify bad amplifiers, saturated and suspect pixels.
1795 
1796  Parameters
1797  ----------
1798  ccdExposure : `lsst.afw.image.Exposure`
1799  Input exposure to be masked.
1800  amp : `lsst.afw.table.AmpInfoCatalog`
1801  Catalog of parameters defining the amplifier on this
1802  exposure to mask.
1803  defects : `lsst.meas.algorithms.Defects`
1804  List of defects. Used to determine if the entire
1805  amplifier is bad.
1806 
1807  Returns
1808  -------
1809  badAmp : `Bool`
1810  If this is true, the entire amplifier area is covered by
1811  defects and unusable.
1812 
1813  """
1814  maskedImage = ccdExposure.getMaskedImage()
1815 
1816  badAmp = False
1817 
1818  # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1819  # comparison with current defects definition.
1820  if defects is not None:
1821  badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1822 
1823  # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1824  # association with pixels in current ccdExposure).
1825  if badAmp:
1826  dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1827  afwImage.PARENT)
1828  maskView = dataView.getMask()
1829  maskView |= maskView.getPlaneBitMask("BAD")
1830  del maskView
1831  return badAmp
1832 
1833  # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1834  # Saturation and suspect pixels can be masked now, though.
1835  limits = dict()
1836  if self.config.doSaturation and not badAmp:
1837  limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1838  if self.config.doSuspect and not badAmp:
1839  limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1840  if math.isfinite(self.config.saturation):
1841  limits.update({self.config.saturatedMaskName: self.config.saturation})
1842 
1843  for maskName, maskThreshold in limits.items():
1844  if not math.isnan(maskThreshold):
1845  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1846  isrFunctions.makeThresholdMask(
1847  maskedImage=dataView,
1848  threshold=maskThreshold,
1849  growFootprints=0,
1850  maskName=maskName
1851  )
1852 
1853  # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1854  maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1855  afwImage.PARENT)
1856  maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1857  self.config.suspectMaskName])
1858  if numpy.all(maskView.getArray() & maskVal > 0):
1859  badAmp = True
1860  maskView |= maskView.getPlaneBitMask("BAD")
1861 
1862  return badAmp
1863 
1864  def overscanCorrection(self, ccdExposure, amp):
1865  """Apply overscan correction in place.
1866 
1867  This method does initial pixel rejection of the overscan
1868  region. The overscan can also be optionally segmented to
1869  allow for discontinuous overscan responses to be fit
1870  separately. The actual overscan subtraction is performed by
1871  the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1872  which is called here after the amplifier is preprocessed.
1873 
1874  Parameters
1875  ----------
1876  ccdExposure : `lsst.afw.image.Exposure`
1877  Exposure to have overscan correction performed.
1878  amp : `lsst.afw.table.AmpInfoCatalog`
1879  The amplifier to consider while correcting the overscan.
1880 
1881  Returns
1882  -------
1883  overscanResults : `lsst.pipe.base.Struct`
1884  Result struct with components:
1885  - ``imageFit`` : scalar or `lsst.afw.image.Image`
1886  Value or fit subtracted from the amplifier image data.
1887  - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1888  Value or fit subtracted from the overscan image data.
1889  - ``overscanImage`` : `lsst.afw.image.Image`
1890  Image of the overscan region with the overscan
1891  correction applied. This quantity is used to estimate
1892  the amplifier read noise empirically.
1893 
1894  Raises
1895  ------
1896  RuntimeError
1897  Raised if the ``amp`` does not contain raw pixel information.
1898 
1899  See Also
1900  --------
1901  lsst.ip.isr.isrFunctions.overscanCorrection
1902  """
1903  if amp.getRawHorizontalOverscanBBox().isEmpty():
1904  self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1905  return None
1906 
1907  statControl = afwMath.StatisticsControl()
1908  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1909 
1910  # Determine the bounding boxes
1911  dataBBox = amp.getRawDataBBox()
1912  oscanBBox = amp.getRawHorizontalOverscanBBox()
1913  dx0 = 0
1914  dx1 = 0
1915 
1916  prescanBBox = amp.getRawPrescanBBox()
1917  if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1918  dx0 += self.config.overscanNumLeadingColumnsToSkip
1919  dx1 -= self.config.overscanNumTrailingColumnsToSkip
1920  else:
1921  dx0 += self.config.overscanNumTrailingColumnsToSkip
1922  dx1 -= self.config.overscanNumLeadingColumnsToSkip
1923 
1924  # Determine if we need to work on subregions of the amplifier and overscan.
1925  imageBBoxes = []
1926  overscanBBoxes = []
1927 
1928  if ((self.config.overscanBiasJump
1929  and self.config.overscanBiasJumpLocation)
1930  and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
1931  and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1932  self.config.overscanBiasJumpDevices)):
1933  if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
1934  yLower = self.config.overscanBiasJumpLocation
1935  yUpper = dataBBox.getHeight() - yLower
1936  else:
1937  yUpper = self.config.overscanBiasJumpLocation
1938  yLower = dataBBox.getHeight() - yUpper
1939 
1940  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1941  lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1942  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1943  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1944  yLower)))
1945 
1946  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1947  lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1948  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1949  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1950  yUpper)))
1951  else:
1952  imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1953  lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1954  overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1955  lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1956  oscanBBox.getHeight())))
1957 
1958  # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1959  for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1960  ampImage = ccdExposure.maskedImage[imageBBox]
1961  overscanImage = ccdExposure.maskedImage[overscanBBox]
1962 
1963  overscanArray = overscanImage.image.array
1964  median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1965  bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1966  overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1967 
1968  statControl = afwMath.StatisticsControl()
1969  statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1970 
1971  overscanResults = self.overscan.run(ampImage.getImage(), overscanImage)
1972 
1973  # Measure average overscan levels and record them in the metadata.
1974  levelStat = afwMath.MEDIAN
1975  sigmaStat = afwMath.STDEVCLIP
1976 
1977  sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1978  self.config.qa.flatness.nIter)
1979  metadata = ccdExposure.getMetadata()
1980  ampNum = amp.getName()
1981  # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1982  if isinstance(overscanResults.overscanFit, float):
1983  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1984  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1985  else:
1986  stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1987  metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1988  metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1989 
1990  return overscanResults
1991 
1992  def updateVariance(self, ampExposure, amp, overscanImage=None):
1993  """Set the variance plane using the amplifier gain and read noise
1994 
1995  The read noise is calculated from the ``overscanImage`` if the
1996  ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1997  the value from the amplifier data is used.
1998 
1999  Parameters
2000  ----------
2001  ampExposure : `lsst.afw.image.Exposure`
2002  Exposure to process.
2003  amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2004  Amplifier detector data.
2005  overscanImage : `lsst.afw.image.MaskedImage`, optional.
2006  Image of overscan, required only for empirical read noise.
2007 
2008  See also
2009  --------
2010  lsst.ip.isr.isrFunctions.updateVariance
2011  """
2012  maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2013  gain = amp.getGain()
2014 
2015  if math.isnan(gain):
2016  gain = 1.0
2017  self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2018  elif gain <= 0:
2019  patchedGain = 1.0
2020  self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
2021  amp.getName(), gain, patchedGain)
2022  gain = patchedGain
2023 
2024  if self.config.doEmpiricalReadNoise and overscanImage is None:
2025  self.log.info("Overscan is none for EmpiricalReadNoise.")
2026 
2027  if self.config.doEmpiricalReadNoise and overscanImage is not None:
2028  stats = afwMath.StatisticsControl()
2029  stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2030  readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2031  self.log.info("Calculated empirical read noise for amp %s: %f.",
2032  amp.getName(), readNoise)
2033  else:
2034  readNoise = amp.getReadNoise()
2035 
2036  isrFunctions.updateVariance(
2037  maskedImage=ampExposure.getMaskedImage(),
2038  gain=gain,
2039  readNoise=readNoise,
2040  )
2041 
2042  def darkCorrection(self, exposure, darkExposure, invert=False):
2043  """!Apply dark correction in place.
2044 
2045  Parameters
2046  ----------
2047  exposure : `lsst.afw.image.Exposure`
2048  Exposure to process.
2049  darkExposure : `lsst.afw.image.Exposure`
2050  Dark exposure of the same size as ``exposure``.
2051  invert : `Bool`, optional
2052  If True, re-add the dark to an already corrected image.
2053 
2054  Raises
2055  ------
2056  RuntimeError
2057  Raised if either ``exposure`` or ``darkExposure`` do not
2058  have their dark time defined.
2059 
2060  See Also
2061  --------
2062  lsst.ip.isr.isrFunctions.darkCorrection
2063  """
2064  expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2065  if math.isnan(expScale):
2066  raise RuntimeError("Exposure darktime is NAN.")
2067  if darkExposure.getInfo().getVisitInfo() is not None \
2068  and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2069  darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2070  else:
2071  # DM-17444: darkExposure.getInfo.getVisitInfo() is None
2072  # so getDarkTime() does not exist.
2073  self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2074  darkScale = 1.0
2075 
2076  isrFunctions.darkCorrection(
2077  maskedImage=exposure.getMaskedImage(),
2078  darkMaskedImage=darkExposure.getMaskedImage(),
2079  expScale=expScale,
2080  darkScale=darkScale,
2081  invert=invert,
2082  trimToFit=self.config.doTrimToMatchCalib
2083  )
2084 
2085  def doLinearize(self, detector):
2086  """!Check if linearization is needed for the detector cameraGeom.
2087 
2088  Checks config.doLinearize and the linearity type of the first
2089  amplifier.
2090 
2091  Parameters
2092  ----------
2093  detector : `lsst.afw.cameraGeom.Detector`
2094  Detector to get linearity type from.
2095 
2096  Returns
2097  -------
2098  doLinearize : `Bool`
2099  If True, linearization should be performed.
2100  """
2101  return self.config.doLinearize and \
2102  detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2103 
2104  def flatCorrection(self, exposure, flatExposure, invert=False):
2105  """!Apply flat correction in place.
2106 
2107  Parameters
2108  ----------
2109  exposure : `lsst.afw.image.Exposure`
2110  Exposure to process.
2111  flatExposure : `lsst.afw.image.Exposure`
2112  Flat exposure of the same size as ``exposure``.
2113  invert : `Bool`, optional
2114  If True, unflatten an already flattened image.
2115 
2116  See Also
2117  --------
2118  lsst.ip.isr.isrFunctions.flatCorrection
2119  """
2120  isrFunctions.flatCorrection(
2121  maskedImage=exposure.getMaskedImage(),
2122  flatMaskedImage=flatExposure.getMaskedImage(),
2123  scalingType=self.config.flatScalingType,
2124  userScale=self.config.flatUserScale,
2125  invert=invert,
2126  trimToFit=self.config.doTrimToMatchCalib
2127  )
2128 
2129  def saturationDetection(self, exposure, amp):
2130  """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2131 
2132  Parameters
2133  ----------
2134  exposure : `lsst.afw.image.Exposure`
2135  Exposure to process. Only the amplifier DataSec is processed.
2136  amp : `lsst.afw.table.AmpInfoCatalog`
2137  Amplifier detector data.
2138 
2139  See Also
2140  --------
2141  lsst.ip.isr.isrFunctions.makeThresholdMask
2142  """
2143  if not math.isnan(amp.getSaturation()):
2144  maskedImage = exposure.getMaskedImage()
2145  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2146  isrFunctions.makeThresholdMask(
2147  maskedImage=dataView,
2148  threshold=amp.getSaturation(),
2149  growFootprints=0,
2150  maskName=self.config.saturatedMaskName,
2151  )
2152 
2153  def saturationInterpolation(self, exposure):
2154  """!Interpolate over saturated pixels, in place.
2155 
2156  This method should be called after `saturationDetection`, to
2157  ensure that the saturated pixels have been identified in the
2158  SAT mask. It should also be called after `assembleCcd`, since
2159  saturated regions may cross amplifier boundaries.
2160 
2161  Parameters
2162  ----------
2163  exposure : `lsst.afw.image.Exposure`
2164  Exposure to process.
2165 
2166  See Also
2167  --------
2168  lsst.ip.isr.isrTask.saturationDetection
2169  lsst.ip.isr.isrFunctions.interpolateFromMask
2170  """
2171  isrFunctions.interpolateFromMask(
2172  maskedImage=exposure.getMaskedImage(),
2173  fwhm=self.config.fwhm,
2174  growSaturatedFootprints=self.config.growSaturationFootprintSize,
2175  maskNameList=list(self.config.saturatedMaskName),
2176  )
2177 
2178  def suspectDetection(self, exposure, amp):
2179  """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2180 
2181  Parameters
2182  ----------
2183  exposure : `lsst.afw.image.Exposure`
2184  Exposure to process. Only the amplifier DataSec is processed.
2185  amp : `lsst.afw.table.AmpInfoCatalog`
2186  Amplifier detector data.
2187 
2188  See Also
2189  --------
2190  lsst.ip.isr.isrFunctions.makeThresholdMask
2191 
2192  Notes
2193  -----
2194  Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2195  This is intended to indicate pixels that may be affected by unknown systematics;
2196  for example if non-linearity corrections above a certain level are unstable
2197  then that would be a useful value for suspectLevel. A value of `nan` indicates
2198  that no such level exists and no pixels are to be masked as suspicious.
2199  """
2200  suspectLevel = amp.getSuspectLevel()
2201  if math.isnan(suspectLevel):
2202  return
2203 
2204  maskedImage = exposure.getMaskedImage()
2205  dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2206  isrFunctions.makeThresholdMask(
2207  maskedImage=dataView,
2208  threshold=suspectLevel,
2209  growFootprints=0,
2210  maskName=self.config.suspectMaskName,
2211  )
2212 
2213  def maskDefect(self, exposure, defectBaseList):
2214  """!Mask defects using mask plane "BAD", in place.
2215 
2216  Parameters
2217  ----------
2218  exposure : `lsst.afw.image.Exposure`
2219  Exposure to process.
2220  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2221  `lsst.afw.image.DefectBase`.
2222  List of defects to mask.
2223 
2224  Notes
2225  -----
2226  Call this after CCD assembly, since defects may cross amplifier boundaries.
2227  """
2228  maskedImage = exposure.getMaskedImage()
2229  if not isinstance(defectBaseList, Defects):
2230  # Promotes DefectBase to Defect
2231  defectList = Defects(defectBaseList)
2232  else:
2233  defectList = defectBaseList
2234  defectList.maskPixels(maskedImage, maskName="BAD")
2235 
2236  def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"):
2237  """!Mask edge pixels with applicable mask plane.
2238 
2239  Parameters
2240  ----------
2241  exposure : `lsst.afw.image.Exposure`
2242  Exposure to process.
2243  numEdgePixels : `int`, optional
2244  Number of edge pixels to mask.
2245  maskPlane : `str`, optional
2246  Mask plane name to use.
2247  """
2248  maskedImage = exposure.getMaskedImage()
2249  maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2250 
2251  if numEdgePixels > 0:
2252  goodBBox = maskedImage.getBBox()
2253  # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2254  goodBBox.grow(-numEdgePixels)
2255  # Mask pixels outside goodBBox
2256  SourceDetectionTask.setEdgeBits(
2257  maskedImage,
2258  goodBBox,
2259  maskBitMask
2260  )
2261 
2262  def maskAndInterpolateDefects(self, exposure, defectBaseList):
2263  """Mask and interpolate defects using mask plane "BAD", in place.
2264 
2265  Parameters
2266  ----------
2267  exposure : `lsst.afw.image.Exposure`
2268  Exposure to process.
2269  defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2270  `lsst.afw.image.DefectBase`.
2271  List of defects to mask and interpolate.
2272 
2273  See Also
2274  --------
2275  lsst.ip.isr.isrTask.maskDefect()
2276  """
2277  self.maskDefect(exposure, defectBaseList)
2278  self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2279  maskPlane="SUSPECT")
2280  isrFunctions.interpolateFromMask(
2281  maskedImage=exposure.getMaskedImage(),
2282  fwhm=self.config.fwhm,
2283  growSaturatedFootprints=0,
2284  maskNameList=["BAD"],
2285  )
2286 
2287  def maskNan(self, exposure):
2288  """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2289 
2290  Parameters
2291  ----------
2292  exposure : `lsst.afw.image.Exposure`
2293  Exposure to process.
2294 
2295  Notes
2296  -----
2297  We mask over all NaNs, including those that are masked with
2298  other bits (because those may or may not be interpolated over
2299  later, and we want to remove all NaNs). Despite this
2300  behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2301  the historical name.
2302  """
2303  maskedImage = exposure.getMaskedImage()
2304 
2305  # Find and mask NaNs
2306  maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2307  maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2308  numNans = maskNans(maskedImage, maskVal)
2309  self.metadata.set("NUMNANS", numNans)
2310  if numNans > 0:
2311  self.log.warn("There were %d unmasked NaNs.", numNans)
2312 
2313  def maskAndInterpolateNan(self, exposure):
2314  """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2315 
2316  Parameters
2317  ----------
2318  exposure : `lsst.afw.image.Exposure`
2319  Exposure to process.
2320 
2321  See Also
2322  --------
2323  lsst.ip.isr.isrTask.maskNan()
2324  """
2325  self.maskNan(exposure)
2326  isrFunctions.interpolateFromMask(
2327  maskedImage=exposure.getMaskedImage(),
2328  fwhm=self.config.fwhm,
2329  growSaturatedFootprints=0,
2330  maskNameList=["UNMASKEDNAN"],
2331  )
2332 
2333  def measureBackground(self, exposure, IsrQaConfig=None):
2334  """Measure the image background in subgrids, for quality control purposes.
2335 
2336  Parameters
2337  ----------
2338  exposure : `lsst.afw.image.Exposure`
2339  Exposure to process.
2340  IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2341  Configuration object containing parameters on which background
2342  statistics and subgrids to use.
2343  """
2344  if IsrQaConfig is not None:
2345  statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2346  IsrQaConfig.flatness.nIter)
2347  maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2348  statsControl.setAndMask(maskVal)
2349  maskedImage = exposure.getMaskedImage()
2350  stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2351  skyLevel = stats.getValue(afwMath.MEDIAN)
2352  skySigma = stats.getValue(afwMath.STDEVCLIP)
2353  self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2354  metadata = exposure.getMetadata()
2355  metadata.set('SKYLEVEL', skyLevel)
2356  metadata.set('SKYSIGMA', skySigma)
2357 
2358  # calcluating flatlevel over the subgrids
2359  stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2360  meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2361  meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2362  nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2363  nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2364  skyLevels = numpy.zeros((nX, nY))
2365 
2366  for j in range(nY):
2367  yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2368  for i in range(nX):
2369  xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2370 
2371  xLLC = xc - meshXHalf
2372  yLLC = yc - meshYHalf
2373  xURC = xc + meshXHalf - 1
2374  yURC = yc + meshYHalf - 1
2375 
2376  bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2377  miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2378 
2379  skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2380 
2381  good = numpy.where(numpy.isfinite(skyLevels))
2382  skyMedian = numpy.median(skyLevels[good])
2383  flatness = (skyLevels[good] - skyMedian) / skyMedian
2384  flatness_rms = numpy.std(flatness)
2385  flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2386 
2387  self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2388  self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2389  nX, nY, flatness_pp, flatness_rms)
2390 
2391  metadata.set('FLATNESS_PP', float(flatness_pp))
2392  metadata.set('FLATNESS_RMS', float(flatness_rms))
2393  metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2394  metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2395  metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2396 
2397  def roughZeroPoint(self, exposure):
2398  """Set an approximate magnitude zero point for the exposure.
2399 
2400  Parameters
2401  ----------
2402  exposure : `lsst.afw.image.Exposure`
2403  Exposure to process.
2404  """
2405  filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2406  if filterName in self.config.fluxMag0T1:
2407  fluxMag0 = self.config.fluxMag0T1[filterName]
2408  else:
2409  self.log.warn("No rough magnitude zero point set for filter %s.", filterName)
2410  fluxMag0 = self.config.defaultFluxMag0T1
2411 
2412  expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2413  if not expTime > 0: # handle NaN as well as <= 0
2414  self.log.warn("Non-positive exposure time; skipping rough zero point.")
2415  return
2416 
2417  self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2418  exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2419 
2420  def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2421  """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2422 
2423  Parameters
2424  ----------
2425  ccdExposure : `lsst.afw.image.Exposure`
2426  Exposure to process.
2427  fpPolygon : `lsst.afw.geom.Polygon`
2428  Polygon in focal plane coordinates.
2429  """
2430  # Get ccd corners in focal plane coordinates
2431  ccd = ccdExposure.getDetector()
2432  fpCorners = ccd.getCorners(FOCAL_PLANE)
2433  ccdPolygon = Polygon(fpCorners)
2434 
2435  # Get intersection of ccd corners with fpPolygon
2436  intersect = ccdPolygon.intersectionSingle(fpPolygon)
2437 
2438  # Transform back to pixel positions and build new polygon
2439  ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2440  validPolygon = Polygon(ccdPoints)
2441  ccdExposure.getInfo().setValidPolygon(validPolygon)
2442 
2443  @contextmanager
2444  def flatContext(self, exp, flat, dark=None):
2445  """Context manager that applies and removes flats and darks,
2446  if the task is configured to apply them.
2447 
2448  Parameters
2449  ----------
2450  exp : `lsst.afw.image.Exposure`
2451  Exposure to process.
2452  flat : `lsst.afw.image.Exposure`
2453  Flat exposure the same size as ``exp``.
2454  dark : `lsst.afw.image.Exposure`, optional
2455  Dark exposure the same size as ``exp``.
2456 
2457  Yields
2458  ------
2459  exp : `lsst.afw.image.Exposure`
2460  The flat and dark corrected exposure.
2461  """
2462  if self.config.doDark and dark is not None:
2463  self.darkCorrection(exp, dark)
2464  if self.config.doFlat:
2465  self.flatCorrection(exp, flat)
2466  try:
2467  yield exp
2468  finally:
2469  if self.config.doFlat:
2470  self.flatCorrection(exp, flat, invert=True)
2471  if self.config.doDark and dark is not None:
2472  self.darkCorrection(exp, dark, invert=True)
2473 
2474  def debugView(self, exposure, stepname):
2475  """Utility function to examine ISR exposure at different stages.
2476 
2477  Parameters
2478  ----------
2479  exposure : `lsst.afw.image.Exposure`
2480  Exposure to view.
2481  stepname : `str`
2482  State of processing to view.
2483  """
2484  frame = getDebugFrame(self._display, stepname)
2485  if frame:
2486  display = getDisplay(frame)
2487  display.scale('asinh', 'zscale')
2488  display.mtv(exposure)
2489  prompt = "Press Enter to continue [c]... "
2490  while True:
2491  ans = input(prompt).lower()
2492  if ans in ("", "c",):
2493  break
2494 
2495 
2496 class FakeAmp(object):
2497  """A Detector-like object that supports returning gain and saturation level
2498 
2499  This is used when the input exposure does not have a detector.
2500 
2501  Parameters
2502  ----------
2503  exposure : `lsst.afw.image.Exposure`
2504  Exposure to generate a fake amplifier for.
2505  config : `lsst.ip.isr.isrTaskConfig`
2506  Configuration to apply to the fake amplifier.
2507  """
2508 
2509  def __init__(self, exposure, config):
2510  self._bbox = exposure.getBBox(afwImage.LOCAL)
2512  self._gain = config.gain
2513  self._readNoise = config.readNoise
2514  self._saturation = config.saturation
2515 
2516  def getBBox(self):
2517  return self._bbox
2518 
2519  def getRawBBox(self):
2520  return self._bbox
2521 
2523  return self._RawHorizontalOverscanBBox
2524 
2525  def getGain(self):
2526  return self._gain
2527 
2528  def getReadNoise(self):
2529  return self._readNoise
2530 
2531  def getSaturation(self):
2532  return self._saturation
2533 
2534  def getSuspectLevel(self):
2535  return float("NaN")
2536 
2537 
2538 class RunIsrConfig(pexConfig.Config):
2539  isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2540 
2541 
2542 class RunIsrTask(pipeBase.CmdLineTask):
2543  """Task to wrap the default IsrTask to allow it to be retargeted.
2544 
2545  The standard IsrTask can be called directly from a command line
2546  program, but doing so removes the ability of the task to be
2547  retargeted. As most cameras override some set of the IsrTask
2548  methods, this would remove those data-specific methods in the
2549  output post-ISR images. This wrapping class fixes the issue,
2550  allowing identical post-ISR images to be generated by both the
2551  processCcd and isrTask code.
2552  """
2553  ConfigClass = RunIsrConfig
2554  _DefaultName = "runIsr"
2555 
2556  def __init__(self, *args, **kwargs):
2557  super().__init__(*args, **kwargs)
2558  self.makeSubtask("isr")
2559 
2560  def runDataRef(self, dataRef):
2561  """
2562  Parameters
2563  ----------
2564  dataRef : `lsst.daf.persistence.ButlerDataRef`
2565  data reference of the detector data to be processed
2566 
2567  Returns
2568  -------
2569  result : `pipeBase.Struct`
2570  Result struct with component:
2571 
2572  - exposure : `lsst.afw.image.Exposure`
2573  Post-ISR processed exposure.
2574  """
2575  return self.isr.runDataRef(dataRef)
lsst::ip::isr.isrTask.IsrTaskConfig.maskListToInterpolate
maskListToInterpolate
Definition: isrTask.py:740
lsst::ip::isr.isrTask.FakeAmp.getRawBBox
def getRawBBox(self)
Definition: isrTask.py:2519
lsst::afw::image
lsst::ip::isr.linearize.Linearizer
Definition: linearize.py:40
lsst::meas::algorithms::detection
lsst::ip::isr.isrTask.IsrTask.debugView
def debugView(self, exposure, stepname)
Definition: isrTask.py:2474
lsst::ip::isr.isrTask.FakeAmp.getReadNoise
def getReadNoise(self)
Definition: isrTask.py:2528
lsst::ip::isr.isrTask.IsrTask.maskAndInterpolateNan
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2313
lsst::ip::isr.isrTask.IsrTaskConfig.validate
def validate(self)
Definition: isrTask.py:832
lsst::ip::isr.isrTask.IsrTaskConnections
Definition: isrTask.py:101
lsst::ip::isr.isrTask.FakeAmp.getRawHorizontalOverscanBBox
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2522
lsst::ip::isr.isrTask.FakeAmp._RawHorizontalOverscanBBox
_RawHorizontalOverscanBBox
Definition: isrTask.py:2511
lsst::ip::isr.isrTask.IsrTask.measureBackground
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2333
lsst::ip::isr.isrTask.IsrTask.maskAmplifier
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1793
lsst::ip::isr.crosstalk.CrosstalkCalib
Definition: crosstalk.py:40
lsst::ip::isr.isrTask.IsrTask.roughZeroPoint
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2397
lsst::ip::isr.isrTask.crosstalkSourceLookup
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
Definition: isrTask.py:62
lsst::ip::isr.isrTask.IsrTask.saturationDetection
def saturationDetection(self, exposure, amp)
Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
Definition: isrTask.py:2129
lsst::ip::isr.isrTask.IsrTask.maskNan
def maskNan(self, exposure)
Definition: isrTask.py:2287
lsst::ip::isr.isrTask.IsrTaskConfig.doNanInterpolation
doNanInterpolation
Definition: isrTask.py:727
lsst::ip::isr.isrTask.IsrTask.runDataRef
def runDataRef(self, sensorRef)
Definition: isrTask.py:1609
lsst::afw::display
lsst::ip::isr.isrTask.IsrTask.suspectDetection
def suspectDetection(self, exposure, amp)
Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
Definition: isrTask.py:2178
lsst::ip::isr.isrTask.IsrTask.maskEdges
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT")
Mask edge pixels with applicable mask plane.
Definition: isrTask.py:2236
lsst::ip::isr.isrTask.FakeAmp.getBBox
def getBBox(self)
Definition: isrTask.py:2516
lsst::ip::isr.isrTask.FakeAmp
Definition: isrTask.py:2496
lsst::ip::isr.isrTask.IsrTask.maskDefect
def maskDefect(self, exposure, defectBaseList)
Mask defects using mask plane "BAD", in place.
Definition: isrTask.py:2213
lsst::ip::isr.isrTask.FakeAmp._readNoise
_readNoise
Definition: isrTask.py:2513
lsst::ip::isr.isrTask.IsrTaskConfig
Definition: isrTask.py:285
lsst::ip::isr.isrTask.IsrTask.flatCorrection
def flatCorrection(self, exposure, flatExposure, invert=False)
Apply flat correction in place.
Definition: isrTask.py:2104
lsst::ip::isr.isrQa.IsrQaConfig
Definition: isrQa.py:58
lsst::ip::isr.isrTask.FakeAmp.getSuspectLevel
def getSuspectLevel(self)
Definition: isrTask.py:2534
lsst::ip::isr.isrTask.IsrTask
Definition: isrTask.py:842
lsst::ip::isr.isrTask.IsrTask.overscanCorrection
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1864
lsst::ip::isr.isrTask.RunIsrTask
Definition: isrTask.py:2542
lsst::ip::isr.isrTask.IsrTask.doLinearize
def doLinearize(self, detector)
Check if linearization is needed for the detector cameraGeom.
Definition: isrTask.py:2085
lsst::ip::isr.isrTask.IsrTaskConfig.doFlat
doFlat
Definition: isrTask.py:636
lsst::ip::isr.isrTask.IsrTask.__init__
def __init__(self, **kwargs)
Definition: isrTask.py:873
lsst::ip::isr::maskNans
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35
lsst::ip::isr.isrTask.IsrTask.darkCorrection
def darkCorrection(self, exposure, darkExposure, invert=False)
Apply dark correction in place.
Definition: isrTask.py:2042
lsst::ip::isr.isrTask.IsrTask.readIsrData
def readIsrData(self, dataRef, rawExposure)
Retrieve necessary frames for instrument signature removal.
Definition: isrTask.py:966
lsst::ip::isr.isrTask.FakeAmp._gain
_gain
Definition: isrTask.py:2512
lsst::ip::isr.isrTask.IsrTask.convertIntToFloat
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1756
lsst::ip::isr.isrTask.IsrTask.flatContext
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2444
lsst::ip::isr.isrTask.RunIsrTask.__init__
def __init__(self, *args, **kwargs)
Definition: isrTask.py:2556
lsst::ip::isr.isrTask.IsrTaskConfig.doSaturationInterpolation
doSaturationInterpolation
Definition: isrTask.py:720
lsst::ip::isr.isrTask.FakeAmp._bbox
_bbox
Definition: isrTask.py:2510
lsst::daf::persistence::butler
lsst::ip::isr.isrTask.IsrTaskConfig.doApplyGains
doApplyGains
Definition: isrTask.py:668
lsst::ip::isr.isrTask.IsrTask.vignettePolygon
vignettePolygon
Definition: isrTask.py:1516
lsst::ip::isr.isrTask.IsrTask.updateVariance
def updateVariance(self, ampExposure, amp, overscanImage=None)
Definition: isrTask.py:1992
lsst::ip::isr.isrTask.IsrTask.maskAndInterpolateDefects
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2262
lsst::ip::isr.isrTask.FakeAmp._saturation
_saturation
Definition: isrTask.py:2514
lsst::ip::isr.isrTask.FakeAmp.__init__
def __init__(self, exposure, config)
Definition: isrTask.py:2509
lsst::ip::isr.isrTask.RunIsrTask.runDataRef
def runDataRef(self, dataRef)
Definition: isrTask.py:2560
lsst::ip::isr.isrTask.FakeAmp.getGain
def getGain(self)
Definition: isrTask.py:2525
lsst::geom
lsst::ip::isr.isrTask.IsrTask.ensureExposure
def ensureExposure(self, inputExp, camera, detectorNum)
Definition: isrTask.py:1708
lsst::afw::cameraGeom
lsst::ip::isr.isrTask.IsrTask.setValidPolygonIntersect
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Set the valid polygon as the intersection of fpPolygon and the ccd corners.
Definition: isrTask.py:2420
lsst::afw::math
Point< int, 2 >
lsst::geom::Box2I
lsst::ip::isr.isrTask.FakeAmp.getSaturation
def getSaturation(self)
Definition: isrTask.py:2531
lsst::ip::isr.isrTask.IsrTask.saturationInterpolation
def saturationInterpolation(self, exposure)
Interpolate over saturated pixels, in place.
Definition: isrTask.py:2153
lsst::ip::isr.isrTask.IsrTask.run
def run(self, ccdExposure, camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Perform instrument signature removal on an exposure.
Definition: isrTask.py:1141
lsst::daf::persistence
lsst::ip::isr.isrTask.IsrTask.runQuantum
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:883
lsst::ip::isr.isrTask.IsrTask.getIsrExposure
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Retrieve a calibration dataset for removing instrument signature.
Definition: isrTask.py:1660
lsst::pipe::base
lsst::meas::algorithms
lsst::ip::isr.isrTask.IsrTaskConnections.__init__
def __init__(self, *config=None)
Definition: isrTask.py:236
Extent< int, 2 >
lsst::pipe::base::connectionTypes
lsst::afw::geom
lsst::ip::isr.isrTask.RunIsrConfig
Definition: isrTask.py:2538