lsst.ip.isr g32debb59f0+967868c433
isrTask.py
Go to the documentation of this file.
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import math
23import numpy
24
25import lsst.geom
26import lsst.afw.image as afwImage
27import lsst.afw.math as afwMath
28import lsst.pex.config as pexConfig
29import lsst.pipe.base as pipeBase
30import lsst.pipe.base.connectionTypes as cT
31
32from contextlib import contextmanager
33from lsstDebug import getDebugFrame
34
35from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36 ReadoutCorner)
37from lsst.afw.display import getDisplay
38from lsst.afw.geom import Polygon
39from lsst.daf.persistence import ButlerDataRef
40from lsst.daf.persistence.butler import NoResults
41from lsst.meas.algorithms.detection import SourceDetectionTask
42
43from . import isrFunctions
44from . import isrQa
45from . import linearize
46from .defects import Defects
47
48from .assembleCcdTask import AssembleCcdTask
49from .crosstalk import CrosstalkTask, CrosstalkCalib
50from .fringe import FringeTask
51from .isr import maskNans
52from .masking import MaskingTask
53from .overscan import OverscanCorrectionTask
54from .straylight import StrayLightTask
55from .vignette import VignetteTask
56from .ampOffset import AmpOffsetTask
57from lsst.daf.butler import DimensionGraph
58
59
60__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
61
62
63def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections):
64 """Lookup function to identify crosstalkSource entries.
65
66 This should return an empty list under most circumstances. Only
67 when inter-chip crosstalk has been identified should this be
68 populated.
69
70 Parameters
71 ----------
72 datasetType : `str`
73 Dataset to lookup.
74 registry : `lsst.daf.butler.Registry`
75 Butler registry to query.
76 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
77 Data id to transform to identify crosstalkSources. The
78 ``detector`` entry will be stripped.
79 collections : `lsst.daf.butler.CollectionSearch`
80 Collections to search through.
81
82 Returns
83 -------
84 results : `list` [`lsst.daf.butler.DatasetRef`]
85 List of datasets that match the query that will be used as
86 crosstalkSources.
87 """
88 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"]))
89 results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId,
90 findFirst=True))
91 # In some contexts, calling `.expanded()` to expand all data IDs in the
92 # query results can be a lot faster because it vectorizes lookups. But in
93 # this case, expandDataId shouldn't need to hit the database at all in the
94 # steady state, because only the detector record is unknown and those are
95 # cached in the registry.
96 return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records)) for ref in results]
97
98
99class IsrTaskConnections(pipeBase.PipelineTaskConnections,
100 dimensions={"instrument", "exposure", "detector"},
101 defaultTemplates={}):
102 ccdExposure = cT.Input(
103 name="raw",
104 doc="Input exposure to process.",
105 storageClass="Exposure",
106 dimensions=["instrument", "exposure", "detector"],
107 )
108 camera = cT.PrerequisiteInput(
109 name="camera",
110 storageClass="Camera",
111 doc="Input camera to construct complete exposures.",
112 dimensions=["instrument"],
113 isCalibration=True,
114 )
115
116 crosstalk = cT.PrerequisiteInput(
117 name="crosstalk",
118 doc="Input crosstalk object",
119 storageClass="CrosstalkCalib",
120 dimensions=["instrument", "detector"],
121 isCalibration=True,
122 minimum=0, # can fall back to cameraGeom
123 )
124 crosstalkSources = cT.PrerequisiteInput(
125 name="isrOverscanCorrected",
126 doc="Overscan corrected input images.",
127 storageClass="Exposure",
128 dimensions=["instrument", "exposure", "detector"],
129 deferLoad=True,
130 multiple=True,
131 lookupFunction=crosstalkSourceLookup,
132 minimum=0, # not needed for all instruments, no config to control this
133 )
134 bias = cT.PrerequisiteInput(
135 name="bias",
136 doc="Input bias calibration.",
137 storageClass="ExposureF",
138 dimensions=["instrument", "detector"],
139 isCalibration=True,
140 )
141 dark = cT.PrerequisiteInput(
142 name='dark',
143 doc="Input dark calibration.",
144 storageClass="ExposureF",
145 dimensions=["instrument", "detector"],
146 isCalibration=True,
147 )
148 flat = cT.PrerequisiteInput(
149 name="flat",
150 doc="Input flat calibration.",
151 storageClass="ExposureF",
152 dimensions=["instrument", "physical_filter", "detector"],
153 isCalibration=True,
154 )
155 ptc = cT.PrerequisiteInput(
156 name="ptc",
157 doc="Input Photon Transfer Curve dataset",
158 storageClass="PhotonTransferCurveDataset",
159 dimensions=["instrument", "detector"],
160 isCalibration=True,
161 )
162 fringes = cT.PrerequisiteInput(
163 name="fringe",
164 doc="Input fringe calibration.",
165 storageClass="ExposureF",
166 dimensions=["instrument", "physical_filter", "detector"],
167 isCalibration=True,
168 minimum=0, # only needed for some bands, even when enabled
169 )
170 strayLightData = cT.PrerequisiteInput(
171 name='yBackground',
172 doc="Input stray light calibration.",
173 storageClass="StrayLightData",
174 dimensions=["instrument", "physical_filter", "detector"],
175 deferLoad=True,
176 isCalibration=True,
177 minimum=0, # only needed for some bands, even when enabled
178 )
179 bfKernel = cT.PrerequisiteInput(
180 name='bfKernel',
181 doc="Input brighter-fatter kernel.",
182 storageClass="NumpyArray",
183 dimensions=["instrument"],
184 isCalibration=True,
185 minimum=0, # can use either bfKernel or newBFKernel
186 )
187 newBFKernel = cT.PrerequisiteInput(
188 name='brighterFatterKernel',
189 doc="Newer complete kernel + gain solutions.",
190 storageClass="BrighterFatterKernel",
191 dimensions=["instrument", "detector"],
192 isCalibration=True,
193 minimum=0, # can use either bfKernel or newBFKernel
194 )
195 defects = cT.PrerequisiteInput(
196 name='defects',
197 doc="Input defect tables.",
198 storageClass="Defects",
199 dimensions=["instrument", "detector"],
200 isCalibration=True,
201 )
202 linearizer = cT.PrerequisiteInput(
203 name='linearizer',
204 storageClass="Linearizer",
205 doc="Linearity correction calibration.",
206 dimensions=["instrument", "detector"],
207 isCalibration=True,
208 minimum=0, # can fall back to cameraGeom
209 )
210 opticsTransmission = cT.PrerequisiteInput(
211 name="transmission_optics",
212 storageClass="TransmissionCurve",
213 doc="Transmission curve due to the optics.",
214 dimensions=["instrument"],
215 isCalibration=True,
216 )
217 filterTransmission = cT.PrerequisiteInput(
218 name="transmission_filter",
219 storageClass="TransmissionCurve",
220 doc="Transmission curve due to the filter.",
221 dimensions=["instrument", "physical_filter"],
222 isCalibration=True,
223 )
224 sensorTransmission = cT.PrerequisiteInput(
225 name="transmission_sensor",
226 storageClass="TransmissionCurve",
227 doc="Transmission curve due to the sensor.",
228 dimensions=["instrument", "detector"],
229 isCalibration=True,
230 )
231 atmosphereTransmission = cT.PrerequisiteInput(
232 name="transmission_atmosphere",
233 storageClass="TransmissionCurve",
234 doc="Transmission curve due to the atmosphere.",
235 dimensions=["instrument"],
236 isCalibration=True,
237 )
238 illumMaskedImage = cT.PrerequisiteInput(
239 name="illum",
240 doc="Input illumination correction.",
241 storageClass="MaskedImageF",
242 dimensions=["instrument", "physical_filter", "detector"],
243 isCalibration=True,
244 )
245
246 outputExposure = cT.Output(
247 name='postISRCCD',
248 doc="Output ISR processed exposure.",
249 storageClass="Exposure",
250 dimensions=["instrument", "exposure", "detector"],
251 )
252 preInterpExposure = cT.Output(
253 name='preInterpISRCCD',
254 doc="Output ISR processed exposure, with pixels left uninterpolated.",
255 storageClass="ExposureF",
256 dimensions=["instrument", "exposure", "detector"],
257 )
258 outputOssThumbnail = cT.Output(
259 name="OssThumb",
260 doc="Output Overscan-subtracted thumbnail image.",
261 storageClass="Thumbnail",
262 dimensions=["instrument", "exposure", "detector"],
263 )
264 outputFlattenedThumbnail = cT.Output(
265 name="FlattenedThumb",
266 doc="Output flat-corrected thumbnail image.",
267 storageClass="Thumbnail",
268 dimensions=["instrument", "exposure", "detector"],
269 )
270
271 def __init__(self, *, config=None):
272 super().__init__(config=config)
273
274 if config.doBias is not True:
275 self.prerequisiteInputs.discard("bias")
276 if config.doLinearize is not True:
277 self.prerequisiteInputs.discard("linearizer")
278 if config.doCrosstalk is not True:
279 self.prerequisiteInputs.discard("crosstalkSources")
280 self.prerequisiteInputs.discard("crosstalk")
281 if config.doBrighterFatter is not True:
282 self.prerequisiteInputs.discard("bfKernel")
283 self.prerequisiteInputs.discard("newBFKernel")
284 if config.doDefect is not True:
285 self.prerequisiteInputs.discard("defects")
286 if config.doDark is not True:
287 self.prerequisiteInputs.discard("dark")
288 if config.doFlat is not True:
289 self.prerequisiteInputs.discard("flat")
290 if config.doFringe is not True:
291 self.prerequisiteInputs.discard("fringe")
292 if config.doStrayLight is not True:
293 self.prerequisiteInputs.discard("strayLightData")
294 if config.usePtcGains is not True and config.usePtcReadNoise is not True:
295 self.prerequisiteInputs.discard("ptc")
296 if config.doAttachTransmissionCurve is not True:
297 self.prerequisiteInputs.discard("opticsTransmission")
298 self.prerequisiteInputs.discard("filterTransmission")
299 self.prerequisiteInputs.discard("sensorTransmission")
300 self.prerequisiteInputs.discard("atmosphereTransmission")
301 if config.doUseOpticsTransmission is not True:
302 self.prerequisiteInputs.discard("opticsTransmission")
303 if config.doUseFilterTransmission is not True:
304 self.prerequisiteInputs.discard("filterTransmission")
305 if config.doUseSensorTransmission is not True:
306 self.prerequisiteInputs.discard("sensorTransmission")
307 if config.doUseAtmosphereTransmission is not True:
308 self.prerequisiteInputs.discard("atmosphereTransmission")
309 if config.doIlluminationCorrection is not True:
310 self.prerequisiteInputs.discard("illumMaskedImage")
311
312 if config.doWrite is not True:
313 self.outputs.discard("outputExposure")
314 self.outputs.discard("preInterpExposure")
315 self.outputs.discard("outputFlattenedThumbnail")
316 self.outputs.discard("outputOssThumbnail")
317 if config.doSaveInterpPixels is not True:
318 self.outputs.discard("preInterpExposure")
319 if config.qa.doThumbnailOss is not True:
320 self.outputs.discard("outputOssThumbnail")
321 if config.qa.doThumbnailFlattened is not True:
322 self.outputs.discard("outputFlattenedThumbnail")
323
324
325class IsrTaskConfig(pipeBase.PipelineTaskConfig,
326 pipelineConnections=IsrTaskConnections):
327 """Configuration parameters for IsrTask.
328
329 Items are grouped in the order in which they are executed by the task.
330 """
331 datasetType = pexConfig.Field(
332 dtype=str,
333 doc="Dataset type for input data; users will typically leave this alone, "
334 "but camera-specific ISR tasks will override it",
335 default="raw",
336 )
337
338 fallbackFilterName = pexConfig.Field(
339 dtype=str,
340 doc="Fallback default filter name for calibrations.",
341 optional=True
342 )
343 useFallbackDate = pexConfig.Field(
344 dtype=bool,
345 doc="Pass observation date when using fallback filter.",
346 default=False,
347 )
348 expectWcs = pexConfig.Field(
349 dtype=bool,
350 default=True,
351 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
352 )
353 fwhm = pexConfig.Field(
354 dtype=float,
355 doc="FWHM of PSF in arcseconds.",
356 default=1.0,
357 )
358 qa = pexConfig.ConfigField(
359 dtype=isrQa.IsrQaConfig,
360 doc="QA related configuration options.",
361 )
362
363 # Image conversion configuration
364 doConvertIntToFloat = pexConfig.Field(
365 dtype=bool,
366 doc="Convert integer raw images to floating point values?",
367 default=True,
368 )
369
370 # Saturated pixel handling.
371 doSaturation = pexConfig.Field(
372 dtype=bool,
373 doc="Mask saturated pixels? NB: this is totally independent of the"
374 " interpolation option - this is ONLY setting the bits in the mask."
375 " To have them interpolated make sure doSaturationInterpolation=True",
376 default=True,
377 )
378 saturatedMaskName = pexConfig.Field(
379 dtype=str,
380 doc="Name of mask plane to use in saturation detection and interpolation",
381 default="SAT",
382 )
383 saturation = pexConfig.Field(
384 dtype=float,
385 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
386 default=float("NaN"),
387 )
388 growSaturationFootprintSize = pexConfig.Field(
389 dtype=int,
390 doc="Number of pixels by which to grow the saturation footprints",
391 default=1,
392 )
393
394 # Suspect pixel handling.
395 doSuspect = pexConfig.Field(
396 dtype=bool,
397 doc="Mask suspect pixels?",
398 default=False,
399 )
400 suspectMaskName = pexConfig.Field(
401 dtype=str,
402 doc="Name of mask plane to use for suspect pixels",
403 default="SUSPECT",
404 )
405 numEdgeSuspect = pexConfig.Field(
406 dtype=int,
407 doc="Number of edge pixels to be flagged as untrustworthy.",
408 default=0,
409 )
410 edgeMaskLevel = pexConfig.ChoiceField(
411 dtype=str,
412 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
413 default="DETECTOR",
414 allowed={
415 'DETECTOR': 'Mask only the edges of the full detector.',
416 'AMP': 'Mask edges of each amplifier.',
417 },
418 )
419
420 # Initial masking options.
421 doSetBadRegions = pexConfig.Field(
422 dtype=bool,
423 doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
424 default=True,
425 )
426 badStatistic = pexConfig.ChoiceField(
427 dtype=str,
428 doc="How to estimate the average value for BAD regions.",
429 default='MEANCLIP',
430 allowed={
431 "MEANCLIP": "Correct using the (clipped) mean of good data",
432 "MEDIAN": "Correct using the median of the good data",
433 },
434 )
435
436 # Overscan subtraction configuration.
437 doOverscan = pexConfig.Field(
438 dtype=bool,
439 doc="Do overscan subtraction?",
440 default=True,
441 )
442 overscan = pexConfig.ConfigurableField(
443 target=OverscanCorrectionTask,
444 doc="Overscan subtraction task for image segments.",
445 )
446 overscanFitType = pexConfig.ChoiceField(
447 dtype=str,
448 doc="The method for fitting the overscan bias level.",
449 default='MEDIAN',
450 allowed={
451 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
452 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
453 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
454 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
455 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
456 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
457 "MEAN": "Correct using the mean of the overscan region",
458 "MEANCLIP": "Correct using a clipped mean of the overscan region",
459 "MEDIAN": "Correct using the median of the overscan region",
460 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
461 },
462 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
463 " This option will no longer be used, and will be removed after v20.")
464 )
465 overscanOrder = pexConfig.Field(
466 dtype=int,
467 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, "
468 "or number of spline knots if overscan fit type is a spline."),
469 default=1,
470 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
471 " This option will no longer be used, and will be removed after v20.")
472 )
473 overscanNumSigmaClip = pexConfig.Field(
474 dtype=float,
475 doc="Rejection threshold (sigma) for collapsing overscan before fit",
476 default=3.0,
477 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
478 " This option will no longer be used, and will be removed after v20.")
479 )
480 overscanIsInt = pexConfig.Field(
481 dtype=bool,
482 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
483 " and overscan.FitType=MEDIAN_PER_ROW.",
484 default=True,
485 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
486 " This option will no longer be used, and will be removed after v20.")
487 )
488 # These options do not get deprecated, as they define how we slice up the
489 # image data.
490 overscanNumLeadingColumnsToSkip = pexConfig.Field(
491 dtype=int,
492 doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
493 default=0,
494 )
495 overscanNumTrailingColumnsToSkip = pexConfig.Field(
496 dtype=int,
497 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
498 default=0,
499 )
500 overscanMaxDev = pexConfig.Field(
501 dtype=float,
502 doc="Maximum deviation from the median for overscan",
503 default=1000.0, check=lambda x: x > 0
504 )
505 overscanBiasJump = pexConfig.Field(
506 dtype=bool,
507 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
508 default=False,
509 )
510 overscanBiasJumpKeyword = pexConfig.Field(
511 dtype=str,
512 doc="Header keyword containing information about devices.",
513 default="NO_SUCH_KEY",
514 )
515 overscanBiasJumpDevices = pexConfig.ListField(
516 dtype=str,
517 doc="List of devices that need piecewise overscan correction.",
518 default=(),
519 )
520 overscanBiasJumpLocation = pexConfig.Field(
521 dtype=int,
522 doc="Location of bias jump along y-axis.",
523 default=0,
524 )
525
526 # Amplifier to CCD assembly configuration
527 doAssembleCcd = pexConfig.Field(
528 dtype=bool,
529 default=True,
530 doc="Assemble amp-level exposures into a ccd-level exposure?"
531 )
532 assembleCcd = pexConfig.ConfigurableField(
533 target=AssembleCcdTask,
534 doc="CCD assembly task",
535 )
536
537 # General calibration configuration.
538 doAssembleIsrExposures = pexConfig.Field(
539 dtype=bool,
540 default=False,
541 doc="Assemble amp-level calibration exposures into ccd-level exposure?"
542 )
543 doTrimToMatchCalib = pexConfig.Field(
544 dtype=bool,
545 default=False,
546 doc="Trim raw data to match calibration bounding boxes?"
547 )
548
549 # Bias subtraction.
550 doBias = pexConfig.Field(
551 dtype=bool,
552 doc="Apply bias frame correction?",
553 default=True,
554 )
555 biasDataProductName = pexConfig.Field(
556 dtype=str,
557 doc="Name of the bias data product",
558 default="bias",
559 )
560 doBiasBeforeOverscan = pexConfig.Field(
561 dtype=bool,
562 doc="Reverse order of overscan and bias correction.",
563 default=False
564 )
565
566 # Variance construction
567 doVariance = pexConfig.Field(
568 dtype=bool,
569 doc="Calculate variance?",
570 default=True
571 )
572 gain = pexConfig.Field(
573 dtype=float,
574 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
575 default=float("NaN"),
576 )
577 readNoise = pexConfig.Field(
578 dtype=float,
579 doc="The read noise to use if no Detector is present in the Exposure",
580 default=0.0,
581 )
582 doEmpiricalReadNoise = pexConfig.Field(
583 dtype=bool,
584 default=False,
585 doc="Calculate empirical read noise instead of value from AmpInfo data?"
586 )
587 usePtcReadNoise = pexConfig.Field(
588 dtype=bool,
589 default=False,
590 doc="Use readnoise values from the Photon Transfer Curve?"
591 )
592 maskNegativeVariance = pexConfig.Field(
593 dtype=bool,
594 default=True,
595 doc="Mask pixels that claim a negative variance? This likely indicates a failure "
596 "in the measurement of the overscan at an edge due to the data falling off faster "
597 "than the overscan model can account for it."
598 )
599 negativeVarianceMaskName = pexConfig.Field(
600 dtype=str,
601 default="BAD",
602 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
603 )
604 # Linearization.
605 doLinearize = pexConfig.Field(
606 dtype=bool,
607 doc="Correct for nonlinearity of the detector's response?",
608 default=True,
609 )
610
611 # Crosstalk.
612 doCrosstalk = pexConfig.Field(
613 dtype=bool,
614 doc="Apply intra-CCD crosstalk correction?",
615 default=False,
616 )
617 doCrosstalkBeforeAssemble = pexConfig.Field(
618 dtype=bool,
619 doc="Apply crosstalk correction before CCD assembly, and before trimming?",
620 default=False,
621 )
622 crosstalk = pexConfig.ConfigurableField(
623 target=CrosstalkTask,
624 doc="Intra-CCD crosstalk correction",
625 )
626
627 # Masking options.
628 doDefect = pexConfig.Field(
629 dtype=bool,
630 doc="Apply correction for CCD defects, e.g. hot pixels?",
631 default=True,
632 )
633 doNanMasking = pexConfig.Field(
634 dtype=bool,
635 doc="Mask non-finite (NAN, inf) pixels?",
636 default=True,
637 )
638 doWidenSaturationTrails = pexConfig.Field(
639 dtype=bool,
640 doc="Widen bleed trails based on their width?",
641 default=True
642 )
643
644 # Brighter-Fatter correction.
645 doBrighterFatter = pexConfig.Field(
646 dtype=bool,
647 default=False,
648 doc="Apply the brighter-fatter correction?"
649 )
650 brighterFatterLevel = pexConfig.ChoiceField(
651 dtype=str,
652 default="DETECTOR",
653 doc="The level at which to correct for brighter-fatter.",
654 allowed={
655 "AMP": "Every amplifier treated separately.",
656 "DETECTOR": "One kernel per detector",
657 }
658 )
659 brighterFatterMaxIter = pexConfig.Field(
660 dtype=int,
661 default=10,
662 doc="Maximum number of iterations for the brighter-fatter correction"
663 )
664 brighterFatterThreshold = pexConfig.Field(
665 dtype=float,
666 default=1000,
667 doc="Threshold used to stop iterating the brighter-fatter correction. It is the "
668 "absolute value of the difference between the current corrected image and the one "
669 "from the previous iteration summed over all the pixels."
670 )
671 brighterFatterApplyGain = pexConfig.Field(
672 dtype=bool,
673 default=True,
674 doc="Should the gain be applied when applying the brighter-fatter correction?"
675 )
676 brighterFatterMaskListToInterpolate = pexConfig.ListField(
677 dtype=str,
678 doc="List of mask planes that should be interpolated over when applying the brighter-fatter "
679 "correction.",
680 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"],
681 )
682 brighterFatterMaskGrowSize = pexConfig.Field(
683 dtype=int,
684 default=0,
685 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
686 "when brighter-fatter correction is applied."
687 )
688
689 # Dark subtraction.
690 doDark = pexConfig.Field(
691 dtype=bool,
692 doc="Apply dark frame correction?",
693 default=True,
694 )
695 darkDataProductName = pexConfig.Field(
696 dtype=str,
697 doc="Name of the dark data product",
698 default="dark",
699 )
700
701 # Camera-specific stray light removal.
702 doStrayLight = pexConfig.Field(
703 dtype=bool,
704 doc="Subtract stray light in the y-band (due to encoder LEDs)?",
705 default=False,
706 )
707 strayLight = pexConfig.ConfigurableField(
708 target=StrayLightTask,
709 doc="y-band stray light correction"
710 )
711
712 # Flat correction.
713 doFlat = pexConfig.Field(
714 dtype=bool,
715 doc="Apply flat field correction?",
716 default=True,
717 )
718 flatDataProductName = pexConfig.Field(
719 dtype=str,
720 doc="Name of the flat data product",
721 default="flat",
722 )
723 flatScalingType = pexConfig.ChoiceField(
724 dtype=str,
725 doc="The method for scaling the flat on the fly.",
726 default='USER',
727 allowed={
728 "USER": "Scale by flatUserScale",
729 "MEAN": "Scale by the inverse of the mean",
730 "MEDIAN": "Scale by the inverse of the median",
731 },
732 )
733 flatUserScale = pexConfig.Field(
734 dtype=float,
735 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
736 default=1.0,
737 )
738 doTweakFlat = pexConfig.Field(
739 dtype=bool,
740 doc="Tweak flats to match observed amplifier ratios?",
741 default=False
742 )
743
744 # Amplifier normalization based on gains instead of using flats
745 # configuration.
746 doApplyGains = pexConfig.Field(
747 dtype=bool,
748 doc="Correct the amplifiers for their gains instead of applying flat correction",
749 default=False,
750 )
751 usePtcGains = pexConfig.Field(
752 dtype=bool,
753 doc="Use the gain values from the Photon Transfer Curve?",
754 default=False,
755 )
756 normalizeGains = pexConfig.Field(
757 dtype=bool,
758 doc="Normalize all the amplifiers in each CCD to have the same median value.",
759 default=False,
760 )
761
762 # Fringe correction.
763 doFringe = pexConfig.Field(
764 dtype=bool,
765 doc="Apply fringe correction?",
766 default=True,
767 )
768 fringe = pexConfig.ConfigurableField(
769 target=FringeTask,
770 doc="Fringe subtraction task",
771 )
772 fringeAfterFlat = pexConfig.Field(
773 dtype=bool,
774 doc="Do fringe subtraction after flat-fielding?",
775 default=True,
776 )
777
778 # Amp offset correction.
779 doAmpOffset = pexConfig.Field(
780 doc="Calculate and apply amp offset corrections?",
781 dtype=bool,
782 default=False,
783 )
784 ampOffset = pexConfig.ConfigurableField(
785 doc="Amp offset correction task.",
786 target=AmpOffsetTask,
787 )
788
789 # Initial CCD-level background statistics options.
790 doMeasureBackground = pexConfig.Field(
791 dtype=bool,
792 doc="Measure the background level on the reduced image?",
793 default=False,
794 )
795
796 # Camera-specific masking configuration.
797 doCameraSpecificMasking = pexConfig.Field(
798 dtype=bool,
799 doc="Mask camera-specific bad regions?",
800 default=False,
801 )
802 masking = pexConfig.ConfigurableField(
803 target=MaskingTask,
804 doc="Masking task."
805 )
806
807 # Interpolation options.
808 doInterpolate = pexConfig.Field(
809 dtype=bool,
810 doc="Interpolate masked pixels?",
811 default=True,
812 )
813 doSaturationInterpolation = pexConfig.Field(
814 dtype=bool,
815 doc="Perform interpolation over pixels masked as saturated?"
816 " NB: This is independent of doSaturation; if that is False this plane"
817 " will likely be blank, resulting in a no-op here.",
818 default=True,
819 )
820 doNanInterpolation = pexConfig.Field(
821 dtype=bool,
822 doc="Perform interpolation over pixels masked as NaN?"
823 " NB: This is independent of doNanMasking; if that is False this plane"
824 " will likely be blank, resulting in a no-op here.",
825 default=True,
826 )
827 doNanInterpAfterFlat = pexConfig.Field(
828 dtype=bool,
829 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
830 "also have to interpolate them before flat-fielding."),
831 default=False,
832 )
833 maskListToInterpolate = pexConfig.ListField(
834 dtype=str,
835 doc="List of mask planes that should be interpolated.",
836 default=['SAT', 'BAD'],
837 )
838 doSaveInterpPixels = pexConfig.Field(
839 dtype=bool,
840 doc="Save a copy of the pre-interpolated pixel values?",
841 default=False,
842 )
843
844 # Default photometric calibration options.
845 fluxMag0T1 = pexConfig.DictField(
846 keytype=str,
847 itemtype=float,
848 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
849 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
850 ))
851 )
852 defaultFluxMag0T1 = pexConfig.Field(
853 dtype=float,
854 doc="Default value for fluxMag0T1 (for an unrecognized filter).",
855 default=pow(10.0, 0.4*28.0)
856 )
857
858 # Vignette correction configuration.
859 doVignette = pexConfig.Field(
860 dtype=bool,
861 doc="Apply vignetting parameters?",
862 default=False,
863 )
864 vignette = pexConfig.ConfigurableField(
865 target=VignetteTask,
866 doc="Vignetting task.",
867 )
868
869 # Transmission curve configuration.
870 doAttachTransmissionCurve = pexConfig.Field(
871 dtype=bool,
872 default=False,
873 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
874 )
875 doUseOpticsTransmission = pexConfig.Field(
876 dtype=bool,
877 default=True,
878 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
879 )
880 doUseFilterTransmission = pexConfig.Field(
881 dtype=bool,
882 default=True,
883 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
884 )
885 doUseSensorTransmission = pexConfig.Field(
886 dtype=bool,
887 default=True,
888 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
889 )
890 doUseAtmosphereTransmission = pexConfig.Field(
891 dtype=bool,
892 default=True,
893 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
894 )
895
896 # Illumination correction.
897 doIlluminationCorrection = pexConfig.Field(
898 dtype=bool,
899 default=False,
900 doc="Perform illumination correction?"
901 )
902 illuminationCorrectionDataProductName = pexConfig.Field(
903 dtype=str,
904 doc="Name of the illumination correction data product.",
905 default="illumcor",
906 )
907 illumScale = pexConfig.Field(
908 dtype=float,
909 doc="Scale factor for the illumination correction.",
910 default=1.0,
911 )
912 illumFilters = pexConfig.ListField(
913 dtype=str,
914 default=[],
915 doc="Only perform illumination correction for these filters."
916 )
917
918 # Write the outputs to disk. If ISR is run as a subtask, this may not
919 # be needed.
920 doWrite = pexConfig.Field(
921 dtype=bool,
922 doc="Persist postISRCCD?",
923 default=True,
924 )
925
926 def validate(self):
927 super().validate()
928 if self.doFlatdoFlat and self.doApplyGainsdoApplyGains:
929 raise ValueError("You may not specify both doFlat and doApplyGains")
930 if self.doBiasBeforeOverscandoBiasBeforeOverscan and self.doTrimToMatchCalibdoTrimToMatchCalib:
931 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
932 if self.doSaturationInterpolationdoSaturationInterpolation and self.saturatedMaskNamesaturatedMaskName not in self.maskListToInterpolatemaskListToInterpolate:
933 self.maskListToInterpolatemaskListToInterpolate.append(self.saturatedMaskNamesaturatedMaskName)
934 if not self.doSaturationInterpolationdoSaturationInterpolation and self.saturatedMaskNamesaturatedMaskName in self.maskListToInterpolatemaskListToInterpolate:
935 self.maskListToInterpolatemaskListToInterpolate.remove(self.saturatedMaskNamesaturatedMaskName)
936 if self.doNanInterpolationdoNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolatemaskListToInterpolate:
937 self.maskListToInterpolatemaskListToInterpolate.append("UNMASKEDNAN")
938
939
940class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
941 """Apply common instrument signature correction algorithms to a raw frame.
942
943 The process for correcting imaging data is very similar from
944 camera to camera. This task provides a vanilla implementation of
945 doing these corrections, including the ability to turn certain
946 corrections off if they are not needed. The inputs to the primary
947 method, `run()`, are a raw exposure to be corrected and the
948 calibration data products. The raw input is a single chip sized
949 mosaic of all amps including overscans and other non-science
950 pixels. The method `runDataRef()` identifies and defines the
951 calibration data products, and is intended for use by a
952 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
954 subclassed for different camera, although the most camera specific
955 methods have been split into subtasks that can be redirected
956 appropriately.
957
958 The __init__ method sets up the subtasks for ISR processing, using
959 the defaults from `lsst.ip.isr`.
960
961 Parameters
962 ----------
963 args : `list`
964 Positional arguments passed to the Task constructor.
965 None used at this time.
966 kwargs : `dict`, optional
967 Keyword arguments passed on to the Task constructor.
968 None used at this time.
969 """
970 ConfigClass = IsrTaskConfig
971 _DefaultName = "isr"
972
973 def __init__(self, **kwargs):
974 super().__init__(**kwargs)
975 self.makeSubtask("assembleCcd")
976 self.makeSubtask("crosstalk")
977 self.makeSubtask("strayLight")
978 self.makeSubtask("fringe")
979 self.makeSubtask("masking")
980 self.makeSubtask("overscan")
981 self.makeSubtask("vignette")
982 self.makeSubtask("ampOffset")
983
984 def runQuantum(self, butlerQC, inputRefs, outputRefs):
985 inputs = butlerQC.get(inputRefs)
986
987 try:
988 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
989 except Exception as e:
990 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
991 (inputRefs, e))
992
993 inputs['isGen3'] = True
994
995 detector = inputs['ccdExposure'].getDetector()
996
997 if self.config.doCrosstalk is True:
998 # Crosstalk sources need to be defined by the pipeline
999 # yaml if they exist.
1000 if 'crosstalk' in inputs and inputs['crosstalk'] is not None:
1001 if not isinstance(inputs['crosstalk'], CrosstalkCalib):
1002 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk'])
1003 else:
1004 coeffVector = (self.config.crosstalk.crosstalkValues
1005 if self.config.crosstalk.useConfigCoefficients else None)
1006 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
1007 inputs['crosstalk'] = crosstalkCalib
1008 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0:
1009 if 'crosstalkSources' not in inputs:
1010 self.log.warning("No crosstalkSources found for chip with interChip terms!")
1011
1012 if self.doLinearizedoLinearize(detector) is True:
1013 if 'linearizer' in inputs:
1014 if isinstance(inputs['linearizer'], dict):
1015 linearizer = linearize.Linearizer(detector=detector, log=self.log)
1016 linearizer.fromYaml(inputs['linearizer'])
1017 self.log.warning("Dictionary linearizers will be deprecated in DM-28741.")
1018 elif isinstance(inputs['linearizer'], numpy.ndarray):
1019 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None),
1020 detector=detector,
1021 log=self.log)
1022 self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.")
1023 else:
1024 linearizer = inputs['linearizer']
1025 linearizer.log = self.log
1026 inputs['linearizer'] = linearizer
1027 else:
1028 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log)
1029 self.log.warning("Constructing linearizer from cameraGeom information.")
1030
1031 if self.config.doDefect is True:
1032 if "defects" in inputs and inputs['defects'] is not None:
1033 # defects is loaded as a BaseCatalog with columns
1034 # x0, y0, width, height. Masking expects a list of defects
1035 # defined by their bounding box
1036 if not isinstance(inputs["defects"], Defects):
1037 inputs["defects"] = Defects.fromTable(inputs["defects"])
1038
1039 # Load the correct style of brighter-fatter kernel, and repack
1040 # the information as a numpy array.
1041 if self.config.doBrighterFatter:
1042 brighterFatterKernel = inputs.pop('newBFKernel', None)
1043 if brighterFatterKernel is None:
1044 brighterFatterKernel = inputs.get('bfKernel', None)
1045
1046 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1047 # This is a ISR calib kernel
1048 detName = detector.getName()
1049 level = brighterFatterKernel.level
1050
1051 # This is expected to be a dictionary of amp-wise gains.
1052 inputs['bfGains'] = brighterFatterKernel.gain
1053 if self.config.brighterFatterLevel == 'DETECTOR':
1054 if level == 'DETECTOR':
1055 if detName in brighterFatterKernel.detKernels:
1056 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1057 else:
1058 raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1059 elif level == 'AMP':
1060 self.log.warning("Making DETECTOR level kernel from AMP based brighter "
1061 "fatter kernels.")
1062 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1063 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1064 elif self.config.brighterFatterLevel == 'AMP':
1065 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1066
1067 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
1068 expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
1069 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
1070 expId=expId,
1071 assembler=self.assembleCcd
1072 if self.config.doAssembleIsrExposures else None)
1073 else:
1074 inputs['fringes'] = pipeBase.Struct(fringes=None)
1075
1076 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
1077 if 'strayLightData' not in inputs:
1078 inputs['strayLightData'] = None
1079
1080 outputs = self.runrun(**inputs)
1081 butlerQC.put(outputs, outputRefs)
1082
1083 def readIsrData(self, dataRef, rawExposure):
1084 """Retrieve necessary frames for instrument signature removal.
1085
1086 Pre-fetching all required ISR data products limits the IO
1087 required by the ISR. Any conflict between the calibration data
1088 available and that needed for ISR is also detected prior to
1089 doing processing, allowing it to fail quickly.
1090
1091 Parameters
1092 ----------
1094 Butler reference of the detector data to be processed
1095 rawExposure : `afw.image.Exposure`
1096 The raw exposure that will later be corrected with the
1097 retrieved calibration data; should not be modified in this
1098 method.
1099
1100 Returns
1101 -------
1102 result : `lsst.pipe.base.Struct`
1103 Result struct with components (which may be `None`):
1104 - ``bias``: bias calibration frame (`afw.image.Exposure`)
1105 - ``linearizer``: functor for linearization
1107 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1108 - ``dark``: dark calibration frame (`afw.image.Exposure`)
1109 - ``flat``: flat calibration frame (`afw.image.Exposure`)
1110 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1111 - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1112 - ``fringes``: `lsst.pipe.base.Struct` with components:
1113 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1114 - ``seed``: random seed derived from the ccdExposureId for random
1115 number generator (`uint32`).
1116 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1117 A ``TransmissionCurve`` that represents the throughput of the
1118 optics, to be evaluated in focal-plane coordinates.
1119 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1120 A ``TransmissionCurve`` that represents the throughput of the
1121 filter itself, to be evaluated in focal-plane coordinates.
1122 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1123 A ``TransmissionCurve`` that represents the throughput of the
1124 sensor itself, to be evaluated in post-assembly trimmed
1125 detector coordinates.
1126 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1127 A ``TransmissionCurve`` that represents the throughput of the
1128 atmosphere, assumed to be spatially constant.
1129 - ``strayLightData`` : `object`
1130 An opaque object containing calibration information for
1131 stray-light correction. If `None`, no correction will be
1132 performed.
1133 - ``illumMaskedImage`` : illumination correction image
1135
1136 Raises
1137 ------
1138 NotImplementedError :
1139 Raised if a per-amplifier brighter-fatter kernel is requested by
1140 the configuration.
1141 """
1142 try:
1143 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1144 dateObs = dateObs.toPython().isoformat()
1145 except RuntimeError:
1146 self.log.warning("Unable to identify dateObs for rawExposure.")
1147 dateObs = None
1148
1149 ccd = rawExposure.getDetector()
1150 filterLabel = rawExposure.getFilterLabel()
1151 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1152 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
1153 biasExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.biasDataProductName)
1154 if self.config.doBias else None)
1155 # immediate=True required for functors and linearizers are functors
1156 # see ticket DM-6515
1157 linearizer = (dataRef.get("linearizer", immediate=True)
1158 if self.doLinearizedoLinearize(ccd) else None)
1159 if linearizer is not None and not isinstance(linearizer, numpy.ndarray):
1160 linearizer.log = self.log
1161 if isinstance(linearizer, numpy.ndarray):
1162 linearizer = linearize.Linearizer(table=linearizer, detector=ccd)
1163
1164 crosstalkCalib = None
1165 if self.config.doCrosstalk:
1166 try:
1167 crosstalkCalib = dataRef.get("crosstalk", immediate=True)
1168 except NoResults:
1169 coeffVector = (self.config.crosstalk.crosstalkValues
1170 if self.config.crosstalk.useConfigCoefficients else None)
1171 crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1172 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1173 if self.config.doCrosstalk else None)
1174
1175 darkExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.darkDataProductName)
1176 if self.config.doDark else None)
1177 flatExposure = (self.getIsrExposuregetIsrExposure(dataRef, self.config.flatDataProductName,
1178 dateObs=dateObs)
1179 if self.config.doFlat else None)
1180
1181 brighterFatterKernel = None
1182 brighterFatterGains = None
1183 if self.config.doBrighterFatter is True:
1184 try:
1185 # Use the new-style cp_pipe version of the kernel if it exists
1186 # If using a new-style kernel, always use the self-consistent
1187 # gains, i.e. the ones inside the kernel object itself
1188 brighterFatterKernel = dataRef.get("brighterFatterKernel")
1189 brighterFatterGains = brighterFatterKernel.gain
1190 self.log.info("New style brighter-fatter kernel (brighterFatterKernel) loaded")
1191 except NoResults:
1192 try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
1193 brighterFatterKernel = dataRef.get("bfKernel")
1194 self.log.info("Old style brighter-fatter kernel (bfKernel) loaded")
1195 except NoResults:
1196 brighterFatterKernel = None
1197 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1198 # If the kernel is not an ndarray, it's the cp_pipe version
1199 # so extract the kernel for this detector, or raise an error
1200 if self.config.brighterFatterLevel == 'DETECTOR':
1201 if brighterFatterKernel.detKernels:
1202 brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()]
1203 else:
1204 raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1205 else:
1206 # TODO DM-15631 for implementing this
1207 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1208
1209 defectList = (dataRef.get("defects")
1210 if self.config.doDefect else None)
1211 expId = rawExposure.getInfo().getVisitInfo().getExposureId()
1212 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1213 if self.config.doAssembleIsrExposures else None)
1214 if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1215 else pipeBase.Struct(fringes=None))
1216
1217 if self.config.doAttachTransmissionCurve:
1218 opticsTransmission = (dataRef.get("transmission_optics")
1219 if self.config.doUseOpticsTransmission else None)
1220 filterTransmission = (dataRef.get("transmission_filter")
1221 if self.config.doUseFilterTransmission else None)
1222 sensorTransmission = (dataRef.get("transmission_sensor")
1223 if self.config.doUseSensorTransmission else None)
1224 atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1225 if self.config.doUseAtmosphereTransmission else None)
1226 else:
1227 opticsTransmission = None
1228 filterTransmission = None
1229 sensorTransmission = None
1230 atmosphereTransmission = None
1231
1232 if self.config.doStrayLight:
1233 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1234 else:
1235 strayLightData = None
1236
1237 illumMaskedImage = (self.getIsrExposuregetIsrExposure(dataRef,
1238 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1239 if (self.config.doIlluminationCorrection
1240 and physicalFilter in self.config.illumFilters)
1241 else None)
1242
1243 # Struct should include only kwargs to run()
1244 return pipeBase.Struct(bias=biasExposure,
1245 linearizer=linearizer,
1246 crosstalk=crosstalkCalib,
1247 crosstalkSources=crosstalkSources,
1248 dark=darkExposure,
1249 flat=flatExposure,
1250 bfKernel=brighterFatterKernel,
1251 bfGains=brighterFatterGains,
1252 defects=defectList,
1253 fringes=fringeStruct,
1254 opticsTransmission=opticsTransmission,
1255 filterTransmission=filterTransmission,
1256 sensorTransmission=sensorTransmission,
1257 atmosphereTransmission=atmosphereTransmission,
1258 strayLightData=strayLightData,
1259 illumMaskedImage=illumMaskedImage
1260 )
1261
1262 @pipeBase.timeMethod
1263 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1264 crosstalk=None, crosstalkSources=None,
1265 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1266 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1267 sensorTransmission=None, atmosphereTransmission=None,
1268 detectorNum=None, strayLightData=None, illumMaskedImage=None,
1269 isGen3=False,
1270 ):
1271 """Perform instrument signature removal on an exposure.
1272
1273 Steps included in the ISR processing, in order performed, are:
1274 - saturation and suspect pixel masking
1275 - overscan subtraction
1276 - CCD assembly of individual amplifiers
1277 - bias subtraction
1278 - variance image construction
1279 - linearization of non-linear response
1280 - crosstalk masking
1281 - brighter-fatter correction
1282 - dark subtraction
1283 - fringe correction
1284 - stray light subtraction
1285 - flat correction
1286 - masking of known defects and camera specific features
1287 - vignette calculation
1288 - appending transmission curve and distortion model
1289
1290 Parameters
1291 ----------
1292 ccdExposure : `lsst.afw.image.Exposure`
1293 The raw exposure that is to be run through ISR. The
1294 exposure is modified by this method.
1295 camera : `lsst.afw.cameraGeom.Camera`, optional
1296 The camera geometry for this exposure. Required if
1297 one or more of ``ccdExposure``, ``bias``, ``dark``, or
1298 ``flat`` does not have an associated detector.
1299 bias : `lsst.afw.image.Exposure`, optional
1300 Bias calibration frame.
1301 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1302 Functor for linearization.
1303 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1304 Calibration for crosstalk.
1305 crosstalkSources : `list`, optional
1306 List of possible crosstalk sources.
1307 dark : `lsst.afw.image.Exposure`, optional
1308 Dark calibration frame.
1309 flat : `lsst.afw.image.Exposure`, optional
1310 Flat calibration frame.
1312 Photon transfer curve dataset, with, e.g., gains
1313 and read noise.
1314 bfKernel : `numpy.ndarray`, optional
1315 Brighter-fatter kernel.
1316 bfGains : `dict` of `float`, optional
1317 Gains used to override the detector's nominal gains for the
1318 brighter-fatter correction. A dict keyed by amplifier name for
1319 the detector in question.
1320 defects : `lsst.ip.isr.Defects`, optional
1321 List of defects.
1322 fringes : `lsst.pipe.base.Struct`, optional
1323 Struct containing the fringe correction data, with
1324 elements:
1325 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1326 - ``seed``: random seed derived from the ccdExposureId for random
1327 number generator (`uint32`)
1328 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1329 A ``TransmissionCurve`` that represents the throughput of the,
1330 optics, to be evaluated in focal-plane coordinates.
1331 filterTransmission : `lsst.afw.image.TransmissionCurve`
1332 A ``TransmissionCurve`` that represents the throughput of the
1333 filter itself, to be evaluated in focal-plane coordinates.
1334 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1335 A ``TransmissionCurve`` that represents the throughput of the
1336 sensor itself, to be evaluated in post-assembly trimmed detector
1337 coordinates.
1338 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1339 A ``TransmissionCurve`` that represents the throughput of the
1340 atmosphere, assumed to be spatially constant.
1341 detectorNum : `int`, optional
1342 The integer number for the detector to process.
1343 isGen3 : bool, optional
1344 Flag this call to run() as using the Gen3 butler environment.
1345 strayLightData : `object`, optional
1346 Opaque object containing calibration information for stray-light
1347 correction. If `None`, no correction will be performed.
1348 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1349 Illumination correction image.
1350
1351 Returns
1352 -------
1353 result : `lsst.pipe.base.Struct`
1354 Result struct with component:
1355 - ``exposure`` : `afw.image.Exposure`
1356 The fully ISR corrected exposure.
1357 - ``outputExposure`` : `afw.image.Exposure`
1358 An alias for `exposure`
1359 - ``ossThumb`` : `numpy.ndarray`
1360 Thumbnail image of the exposure after overscan subtraction.
1361 - ``flattenedThumb`` : `numpy.ndarray`
1362 Thumbnail image of the exposure after flat-field correction.
1363
1364 Raises
1365 ------
1366 RuntimeError
1367 Raised if a configuration option is set to True, but the
1368 required calibration data has not been specified.
1369
1370 Notes
1371 -----
1372 The current processed exposure can be viewed by setting the
1373 appropriate lsstDebug entries in the `debug.display`
1374 dictionary. The names of these entries correspond to some of
1375 the IsrTaskConfig Boolean options, with the value denoting the
1376 frame to use. The exposure is shown inside the matching
1377 option check and after the processing of that step has
1378 finished. The steps with debug points are:
1379
1380 doAssembleCcd
1381 doBias
1382 doCrosstalk
1383 doBrighterFatter
1384 doDark
1385 doFringe
1386 doStrayLight
1387 doFlat
1388
1389 In addition, setting the "postISRCCD" entry displays the
1390 exposure after all ISR processing has finished.
1391
1392 """
1393
1394 if isGen3 is True:
1395 # Gen3 currently cannot automatically do configuration overrides.
1396 # DM-15257 looks to discuss this issue.
1397 # Configure input exposures;
1398
1399 ccdExposure = self.ensureExposureensureExposure(ccdExposure, camera, detectorNum)
1400 bias = self.ensureExposureensureExposure(bias, camera, detectorNum)
1401 dark = self.ensureExposureensureExposure(dark, camera, detectorNum)
1402 flat = self.ensureExposureensureExposure(flat, camera, detectorNum)
1403 else:
1404 if isinstance(ccdExposure, ButlerDataRef):
1405 return self.runDataRefrunDataRef(ccdExposure)
1406
1407 ccd = ccdExposure.getDetector()
1408 filterLabel = ccdExposure.getFilterLabel()
1409 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1410
1411 if not ccd:
1412 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1413 ccd = [FakeAmp(ccdExposure, self.config)]
1414
1415 # Validate Input
1416 if self.config.doBias and bias is None:
1417 raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1418 if self.doLinearizedoLinearize(ccd) and linearizer is None:
1419 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1420 if self.config.doBrighterFatter and bfKernel is None:
1421 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1422 if self.config.doDark and dark is None:
1423 raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1424 if self.config.doFlat and flat is None:
1425 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1426 if self.config.doDefect and defects is None:
1427 raise RuntimeError("Must supply defects if config.doDefect=True.")
1428 if (self.config.doFringe and physicalFilter in self.fringe.config.filters
1429 and fringes.fringes is None):
1430 # The `fringes` object needs to be a pipeBase.Struct, as
1431 # we use it as a `dict` for the parameters of
1432 # `FringeTask.run()`. The `fringes.fringes` `list` may
1433 # not be `None` if `doFringe=True`. Otherwise, raise.
1434 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1435 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters
1436 and illumMaskedImage is None):
1437 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1438
1439 # Begin ISR processing.
1440 if self.config.doConvertIntToFloat:
1441 self.log.info("Converting exposure to floating point values.")
1442 ccdExposure = self.convertIntToFloatconvertIntToFloat(ccdExposure)
1443
1444 if self.config.doBias and self.config.doBiasBeforeOverscan:
1445 self.log.info("Applying bias correction.")
1446 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1447 trimToFit=self.config.doTrimToMatchCalib)
1448 self.debugViewdebugView(ccdExposure, "doBias")
1449
1450 # Amplifier level processing.
1451 overscans = []
1452 for amp in ccd:
1453 # if ccdExposure is one amp,
1454 # check for coverage to prevent performing ops multiple times
1455 if ccdExposure.getBBox().contains(amp.getBBox()):
1456 # Check for fully masked bad amplifiers,
1457 # and generate masks for SUSPECT and SATURATED values.
1458 badAmp = self.maskAmplifiermaskAmplifier(ccdExposure, amp, defects)
1459
1460 if self.config.doOverscan and not badAmp:
1461 # Overscan correction on amp-by-amp basis.
1462 overscanResults = self.overscanCorrectionoverscanCorrection(ccdExposure, amp)
1463 self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1464 if overscanResults is not None and \
1465 self.config.qa is not None and self.config.qa.saveStats is True:
1466 if isinstance(overscanResults.overscanFit, float):
1467 qaMedian = overscanResults.overscanFit
1468 qaStdev = float("NaN")
1469 else:
1470 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1471 afwMath.MEDIAN | afwMath.STDEVCLIP)
1472 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1473 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1474
1475 self.metadata.set(f"FIT MEDIAN {amp.getName()}", qaMedian)
1476 self.metadata.set(f"FIT STDEV {amp.getName()}", qaStdev)
1477 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1478 amp.getName(), qaMedian, qaStdev)
1479
1480 # Residuals after overscan correction
1481 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1482 afwMath.MEDIAN | afwMath.STDEVCLIP)
1483 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1484 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1485
1486 self.metadata.set(f"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter)
1487 self.metadata.set(f"RESIDUAL STDEV {amp.getName()}", qaStdevAfter)
1488 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f",
1489 amp.getName(), qaMedianAfter, qaStdevAfter)
1490
1491 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1492 else:
1493 if badAmp:
1494 self.log.warning("Amplifier %s is bad.", amp.getName())
1495 overscanResults = None
1496
1497 overscans.append(overscanResults if overscanResults is not None else None)
1498 else:
1499 self.log.info("Skipped OSCAN for %s.", amp.getName())
1500
1501 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1502 self.log.info("Applying crosstalk correction.")
1503 self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1504 crosstalkSources=crosstalkSources, camera=camera)
1505 self.debugViewdebugView(ccdExposure, "doCrosstalk")
1506
1507 if self.config.doAssembleCcd:
1508 self.log.info("Assembling CCD from amplifiers.")
1509 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1510
1511 if self.config.expectWcs and not ccdExposure.getWcs():
1512 self.log.warning("No WCS found in input exposure.")
1513 self.debugViewdebugView(ccdExposure, "doAssembleCcd")
1514
1515 ossThumb = None
1516 if self.config.qa.doThumbnailOss:
1517 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1518
1519 if self.config.doBias and not self.config.doBiasBeforeOverscan:
1520 self.log.info("Applying bias correction.")
1521 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1522 trimToFit=self.config.doTrimToMatchCalib)
1523 self.debugViewdebugView(ccdExposure, "doBias")
1524
1525 if self.config.doVariance:
1526 for amp, overscanResults in zip(ccd, overscans):
1527 if ccdExposure.getBBox().contains(amp.getBBox()):
1528 self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1529 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1530 if overscanResults is not None:
1531 self.updateVarianceupdateVariance(ampExposure, amp,
1532 overscanImage=overscanResults.overscanImage,
1533 ptcDataset=ptc)
1534 else:
1535 self.updateVarianceupdateVariance(ampExposure, amp,
1536 overscanImage=None,
1537 ptcDataset=ptc)
1538 if self.config.qa is not None and self.config.qa.saveStats is True:
1539 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1540 afwMath.MEDIAN | afwMath.STDEVCLIP)
1541 self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1542 qaStats.getValue(afwMath.MEDIAN))
1543 self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1544 qaStats.getValue(afwMath.STDEVCLIP))
1545 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1546 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1547 qaStats.getValue(afwMath.STDEVCLIP))
1548 if self.config.maskNegativeVariance:
1549 self.maskNegativeVariancemaskNegativeVariance(ccdExposure)
1550
1551 if self.doLinearizedoLinearize(ccd):
1552 self.log.info("Applying linearizer.")
1553 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1554 detector=ccd, log=self.log)
1555
1556 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1557 self.log.info("Applying crosstalk correction.")
1558 self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1559 crosstalkSources=crosstalkSources, isTrimmed=True)
1560 self.debugViewdebugView(ccdExposure, "doCrosstalk")
1561
1562 # Masking block. Optionally mask known defects, NAN/inf pixels,
1563 # widen trails, and do anything else the camera needs. Saturated and
1564 # suspect pixels have already been masked.
1565 if self.config.doDefect:
1566 self.log.info("Masking defects.")
1567 self.maskDefectmaskDefect(ccdExposure, defects)
1568
1569 if self.config.numEdgeSuspect > 0:
1570 self.log.info("Masking edges as SUSPECT.")
1571 self.maskEdgesmaskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1572 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
1573
1574 if self.config.doNanMasking:
1575 self.log.info("Masking non-finite (NAN, inf) value pixels.")
1576 self.maskNanmaskNan(ccdExposure)
1577
1578 if self.config.doWidenSaturationTrails:
1579 self.log.info("Widening saturation trails.")
1580 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1581
1582 if self.config.doCameraSpecificMasking:
1583 self.log.info("Masking regions for camera specific reasons.")
1584 self.masking.run(ccdExposure)
1585
1586 if self.config.doBrighterFatter:
1587 # We need to apply flats and darks before we can interpolate, and
1588 # we need to interpolate before we do B-F, but we do B-F without
1589 # the flats and darks applied so we can work in units of electrons
1590 # or holes. This context manager applies and then removes the darks
1591 # and flats.
1592 #
1593 # We also do not want to interpolate values here, so operate on
1594 # temporary images so we can apply only the BF-correction and roll
1595 # back the interpolation.
1596 interpExp = ccdExposure.clone()
1597 with self.flatContextflatContext(interpExp, flat, dark):
1598 isrFunctions.interpolateFromMask(
1599 maskedImage=interpExp.getMaskedImage(),
1600 fwhm=self.config.fwhm,
1601 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1602 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1603 )
1604 bfExp = interpExp.clone()
1605
1606 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.",
1607 type(bfKernel), type(bfGains))
1608 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1609 self.config.brighterFatterMaxIter,
1610 self.config.brighterFatterThreshold,
1611 self.config.brighterFatterApplyGain,
1612 bfGains)
1613 if bfResults[1] == self.config.brighterFatterMaxIter:
1614 self.log.warning("Brighter-fatter correction did not converge, final difference %f.",
1615 bfResults[0])
1616 else:
1617 self.log.info("Finished brighter-fatter correction in %d iterations.",
1618 bfResults[1])
1619 image = ccdExposure.getMaskedImage().getImage()
1620 bfCorr = bfExp.getMaskedImage().getImage()
1621 bfCorr -= interpExp.getMaskedImage().getImage()
1622 image += bfCorr
1623
1624 # Applying the brighter-fatter correction applies a
1625 # convolution to the science image. At the edges this
1626 # convolution may not have sufficient valid pixels to
1627 # produce a valid correction. Mark pixels within the size
1628 # of the brighter-fatter kernel as EDGE to warn of this
1629 # fact.
1630 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1631 self.maskEdgesmaskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1632 maskPlane="EDGE")
1633
1634 if self.config.brighterFatterMaskGrowSize > 0:
1635 self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1636 for maskPlane in self.config.brighterFatterMaskListToInterpolate:
1637 isrFunctions.growMasks(ccdExposure.getMask(),
1638 radius=self.config.brighterFatterMaskGrowSize,
1639 maskNameList=maskPlane,
1640 maskValue=maskPlane)
1641
1642 self.debugViewdebugView(ccdExposure, "doBrighterFatter")
1643
1644 if self.config.doDark:
1645 self.log.info("Applying dark correction.")
1646 self.darkCorrectiondarkCorrection(ccdExposure, dark)
1647 self.debugViewdebugView(ccdExposure, "doDark")
1648
1649 if self.config.doFringe and not self.config.fringeAfterFlat:
1650 self.log.info("Applying fringe correction before flat.")
1651 self.fringe.run(ccdExposure, **fringes.getDict())
1652 self.debugViewdebugView(ccdExposure, "doFringe")
1653
1654 if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1655 self.log.info("Checking strayLight correction.")
1656 self.strayLight.run(ccdExposure, strayLightData)
1657 self.debugViewdebugView(ccdExposure, "doStrayLight")
1658
1659 if self.config.doFlat:
1660 self.log.info("Applying flat correction.")
1661 self.flatCorrectionflatCorrection(ccdExposure, flat)
1662 self.debugViewdebugView(ccdExposure, "doFlat")
1663
1664 if self.config.doApplyGains:
1665 self.log.info("Applying gain correction instead of flat.")
1666 if self.config.usePtcGains:
1667 self.log.info("Using gains from the Photon Transfer Curve.")
1668 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1669 ptcGains=ptc.gain)
1670 else:
1671 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1672
1673 if self.config.doFringe and self.config.fringeAfterFlat:
1674 self.log.info("Applying fringe correction after flat.")
1675 self.fringe.run(ccdExposure, **fringes.getDict())
1676
1677 if self.config.doVignette:
1678 self.log.info("Constructing Vignette polygon.")
1679 self.vignettePolygonvignettePolygon = self.vignette.run(ccdExposure)
1680
1681 if self.config.vignette.doWriteVignettePolygon:
1682 self.setValidPolygonIntersectsetValidPolygonIntersect(ccdExposure, self.vignettePolygonvignettePolygon)
1683
1684 if self.config.doAttachTransmissionCurve:
1685 self.log.info("Adding transmission curves.")
1686 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1687 filterTransmission=filterTransmission,
1688 sensorTransmission=sensorTransmission,
1689 atmosphereTransmission=atmosphereTransmission)
1690
1691 flattenedThumb = None
1692 if self.config.qa.doThumbnailFlattened:
1693 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1694
1695 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters:
1696 self.log.info("Performing illumination correction.")
1697 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1698 illumMaskedImage, illumScale=self.config.illumScale,
1699 trimToFit=self.config.doTrimToMatchCalib)
1700
1701 preInterpExp = None
1702 if self.config.doSaveInterpPixels:
1703 preInterpExp = ccdExposure.clone()
1704
1705 # Reset and interpolate bad pixels.
1706 #
1707 # Large contiguous bad regions (which should have the BAD mask
1708 # bit set) should have their values set to the image median.
1709 # This group should include defects and bad amplifiers. As the
1710 # area covered by these defects are large, there's little
1711 # reason to expect that interpolation would provide a more
1712 # useful value.
1713 #
1714 # Smaller defects can be safely interpolated after the larger
1715 # regions have had their pixel values reset. This ensures
1716 # that the remaining defects adjacent to bad amplifiers (as an
1717 # example) do not attempt to interpolate extreme values.
1718 if self.config.doSetBadRegions:
1719 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1720 if badPixelCount > 0:
1721 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1722
1723 if self.config.doInterpolate:
1724 self.log.info("Interpolating masked pixels.")
1725 isrFunctions.interpolateFromMask(
1726 maskedImage=ccdExposure.getMaskedImage(),
1727 fwhm=self.config.fwhm,
1728 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1729 maskNameList=list(self.config.maskListToInterpolate)
1730 )
1731
1732 self.roughZeroPointroughZeroPoint(ccdExposure)
1733
1734 # correct for amp offsets within the CCD
1735 if self.config.doAmpOffset:
1736 self.log.info("Correcting amp offsets.")
1737 self.ampOffset.run(ccdExposure)
1738
1739 if self.config.doMeasureBackground:
1740 self.log.info("Measuring background level.")
1741 self.measureBackgroundmeasureBackground(ccdExposure, self.config.qa)
1742
1743 if self.config.qa is not None and self.config.qa.saveStats is True:
1744 for amp in ccd:
1745 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1746 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1747 afwMath.MEDIAN | afwMath.STDEVCLIP)
1748 self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1749 qaStats.getValue(afwMath.MEDIAN))
1750 self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1751 qaStats.getValue(afwMath.STDEVCLIP))
1752 self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1753 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1754 qaStats.getValue(afwMath.STDEVCLIP))
1755
1756 self.debugViewdebugView(ccdExposure, "postISRCCD")
1757
1758 return pipeBase.Struct(
1759 exposure=ccdExposure,
1760 ossThumb=ossThumb,
1761 flattenedThumb=flattenedThumb,
1762
1763 preInterpExposure=preInterpExp,
1764 outputExposure=ccdExposure,
1765 outputOssThumbnail=ossThumb,
1766 outputFlattenedThumbnail=flattenedThumb,
1767 )
1768
1769 @pipeBase.timeMethod
1770 def runDataRef(self, sensorRef):
1771 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1772
1773 This method contains the `CmdLineTask` interface to the ISR
1774 processing. All IO is handled here, freeing the `run()` method
1775 to manage only pixel-level calculations. The steps performed
1776 are:
1777 - Read in necessary detrending/isr/calibration data.
1778 - Process raw exposure in `run()`.
1779 - Persist the ISR-corrected exposure as "postISRCCD" if
1780 config.doWrite=True.
1781
1782 Parameters
1783 ----------
1785 DataRef of the detector data to be processed
1786
1787 Returns
1788 -------
1789 result : `lsst.pipe.base.Struct`
1790 Result struct with component:
1791 - ``exposure`` : `afw.image.Exposure`
1792 The fully ISR corrected exposure.
1793
1794 Raises
1795 ------
1796 RuntimeError
1797 Raised if a configuration option is set to True, but the
1798 required calibration data does not exist.
1799
1800 """
1801 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1802
1803 ccdExposure = sensorRef.get(self.config.datasetType)
1804
1805 camera = sensorRef.get("camera")
1806 isrData = self.readIsrDatareadIsrData(sensorRef, ccdExposure)
1807
1808 result = self.runrun(ccdExposure, camera=camera, **isrData.getDict())
1809
1810 if self.config.doWrite:
1811 sensorRef.put(result.exposure, "postISRCCD")
1812 if result.preInterpExposure is not None:
1813 sensorRef.put(result.preInterpExposure, "postISRCCD_uninterpolated")
1814 if result.ossThumb is not None:
1815 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1816 if result.flattenedThumb is not None:
1817 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1818
1819 return result
1820
1821 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1822 """Retrieve a calibration dataset for removing instrument signature.
1823
1824 Parameters
1825 ----------
1826
1828 DataRef of the detector data to find calibration datasets
1829 for.
1830 datasetType : `str`
1831 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1832 dateObs : `str`, optional
1833 Date of the observation. Used to correct butler failures
1834 when using fallback filters.
1835 immediate : `Bool`
1836 If True, disable butler proxies to enable error handling
1837 within this routine.
1838
1839 Returns
1840 -------
1841 exposure : `lsst.afw.image.Exposure`
1842 Requested calibration frame.
1843
1844 Raises
1845 ------
1846 RuntimeError
1847 Raised if no matching calibration frame can be found.
1848 """
1849 try:
1850 exp = dataRef.get(datasetType, immediate=immediate)
1851 except Exception as exc1:
1852 if not self.config.fallbackFilterName:
1853 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1854 try:
1855 if self.config.useFallbackDate and dateObs:
1856 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1857 dateObs=dateObs, immediate=immediate)
1858 else:
1859 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1860 except Exception as exc2:
1861 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1862 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1863 self.log.warning("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1864
1865 if self.config.doAssembleIsrExposures:
1866 exp = self.assembleCcd.assembleCcd(exp)
1867 return exp
1868
1869 def ensureExposure(self, inputExp, camera=None, detectorNum=None):
1870 """Ensure that the data returned by Butler is a fully constructed exp.
1871
1872 ISR requires exposure-level image data for historical reasons, so if we
1873 did not recieve that from Butler, construct it from what we have,
1874 modifying the input in place.
1875
1876 Parameters
1877 ----------
1878 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`,
1879 or `lsst.afw.image.ImageF`
1880 The input data structure obtained from Butler.
1881 camera : `lsst.afw.cameraGeom.camera`, optional
1882 The camera associated with the image. Used to find the appropriate
1883 detector if detector is not already set.
1884 detectorNum : `int`, optional
1885 The detector in the camera to attach, if the detector is not
1886 already set.
1887
1888 Returns
1889 -------
1890 inputExp : `lsst.afw.image.Exposure`
1891 The re-constructed exposure, with appropriate detector parameters.
1892
1893 Raises
1894 ------
1895 TypeError
1896 Raised if the input data cannot be used to construct an exposure.
1897 """
1898 if isinstance(inputExp, afwImage.DecoratedImageU):
1899 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1900 elif isinstance(inputExp, afwImage.ImageF):
1901 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1902 elif isinstance(inputExp, afwImage.MaskedImageF):
1903 inputExp = afwImage.makeExposure(inputExp)
1904 elif isinstance(inputExp, afwImage.Exposure):
1905 pass
1906 elif inputExp is None:
1907 # Assume this will be caught by the setup if it is a problem.
1908 return inputExp
1909 else:
1910 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1911 (type(inputExp), ))
1912
1913 if inputExp.getDetector() is None:
1914 if camera is None or detectorNum is None:
1915 raise RuntimeError('Must supply both a camera and detector number when using exposures '
1916 'without a detector set.')
1917 inputExp.setDetector(camera[detectorNum])
1918
1919 return inputExp
1920
1921 def convertIntToFloat(self, exposure):
1922 """Convert exposure image from uint16 to float.
1923
1924 If the exposure does not need to be converted, the input is
1925 immediately returned. For exposures that are converted to use
1926 floating point pixels, the variance is set to unity and the
1927 mask to zero.
1928
1929 Parameters
1930 ----------
1931 exposure : `lsst.afw.image.Exposure`
1932 The raw exposure to be converted.
1933
1934 Returns
1935 -------
1936 newexposure : `lsst.afw.image.Exposure`
1937 The input ``exposure``, converted to floating point pixels.
1938
1939 Raises
1940 ------
1941 RuntimeError
1942 Raised if the exposure type cannot be converted to float.
1943
1944 """
1945 if isinstance(exposure, afwImage.ExposureF):
1946 # Nothing to be done
1947 self.log.debug("Exposure already of type float.")
1948 return exposure
1949 if not hasattr(exposure, "convertF"):
1950 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1951
1952 newexposure = exposure.convertF()
1953 newexposure.variance[:] = 1
1954 newexposure.mask[:] = 0x0
1955
1956 return newexposure
1957
1958 def maskAmplifier(self, ccdExposure, amp, defects):
1959 """Identify bad amplifiers, saturated and suspect pixels.
1960
1961 Parameters
1962 ----------
1963 ccdExposure : `lsst.afw.image.Exposure`
1964 Input exposure to be masked.
1966 Catalog of parameters defining the amplifier on this
1967 exposure to mask.
1968 defects : `lsst.ip.isr.Defects`
1969 List of defects. Used to determine if the entire
1970 amplifier is bad.
1971
1972 Returns
1973 -------
1974 badAmp : `Bool`
1975 If this is true, the entire amplifier area is covered by
1976 defects and unusable.
1977
1978 """
1979 maskedImage = ccdExposure.getMaskedImage()
1980
1981 badAmp = False
1982
1983 # Check if entire amp region is defined as a defect
1984 # NB: need to use amp.getBBox() for correct comparison with current
1985 # defects definition.
1986 if defects is not None:
1987 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1988
1989 # In the case of a bad amp, we will set mask to "BAD"
1990 # (here use amp.getRawBBox() for correct association with pixels in
1991 # current ccdExposure).
1992 if badAmp:
1993 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1994 afwImage.PARENT)
1995 maskView = dataView.getMask()
1996 maskView |= maskView.getPlaneBitMask("BAD")
1997 del maskView
1998 return badAmp
1999
2000 # Mask remaining defects after assembleCcd() to allow for defects that
2001 # cross amplifier boundaries. Saturation and suspect pixels can be
2002 # masked now, though.
2003 limits = dict()
2004 if self.config.doSaturation and not badAmp:
2005 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
2006 if self.config.doSuspect and not badAmp:
2007 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
2008 if math.isfinite(self.config.saturation):
2009 limits.update({self.config.saturatedMaskName: self.config.saturation})
2010
2011 for maskName, maskThreshold in limits.items():
2012 if not math.isnan(maskThreshold):
2013 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2014 isrFunctions.makeThresholdMask(
2015 maskedImage=dataView,
2016 threshold=maskThreshold,
2017 growFootprints=0,
2018 maskName=maskName
2019 )
2020
2021 # Determine if we've fully masked this amplifier with SUSPECT and
2022 # SAT pixels.
2023 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
2024 afwImage.PARENT)
2025 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
2026 self.config.suspectMaskName])
2027 if numpy.all(maskView.getArray() & maskVal > 0):
2028 badAmp = True
2029 maskView |= maskView.getPlaneBitMask("BAD")
2030
2031 return badAmp
2032
2033 def overscanCorrection(self, ccdExposure, amp):
2034 """Apply overscan correction in place.
2035
2036 This method does initial pixel rejection of the overscan
2037 region. The overscan can also be optionally segmented to
2038 allow for discontinuous overscan responses to be fit
2039 separately. The actual overscan subtraction is performed by
2040 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
2041 which is called here after the amplifier is preprocessed.
2042
2043 Parameters
2044 ----------
2045 ccdExposure : `lsst.afw.image.Exposure`
2046 Exposure to have overscan correction performed.
2047 amp : `lsst.afw.cameraGeom.Amplifer`
2048 The amplifier to consider while correcting the overscan.
2049
2050 Returns
2051 -------
2052 overscanResults : `lsst.pipe.base.Struct`
2053 Result struct with components:
2054 - ``imageFit`` : scalar or `lsst.afw.image.Image`
2055 Value or fit subtracted from the amplifier image data.
2056 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
2057 Value or fit subtracted from the overscan image data.
2058 - ``overscanImage`` : `lsst.afw.image.Image`
2059 Image of the overscan region with the overscan
2060 correction applied. This quantity is used to estimate
2061 the amplifier read noise empirically.
2062
2063 Raises
2064 ------
2065 RuntimeError
2066 Raised if the ``amp`` does not contain raw pixel information.
2067
2068 See Also
2069 --------
2070 lsst.ip.isr.isrFunctions.overscanCorrection
2071 """
2072 if amp.getRawHorizontalOverscanBBox().isEmpty():
2073 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
2074 return None
2075
2076 statControl = afwMath.StatisticsControl()
2077 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2078
2079 # Determine the bounding boxes
2080 dataBBox = amp.getRawDataBBox()
2081 oscanBBox = amp.getRawHorizontalOverscanBBox()
2082 dx0 = 0
2083 dx1 = 0
2084
2085 prescanBBox = amp.getRawPrescanBBox()
2086 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
2087 dx0 += self.config.overscanNumLeadingColumnsToSkip
2088 dx1 -= self.config.overscanNumTrailingColumnsToSkip
2089 else:
2090 dx0 += self.config.overscanNumTrailingColumnsToSkip
2091 dx1 -= self.config.overscanNumLeadingColumnsToSkip
2092
2093 # Determine if we need to work on subregions of the amplifier
2094 # and overscan.
2095 imageBBoxes = []
2096 overscanBBoxes = []
2097
2098 if ((self.config.overscanBiasJump
2099 and self.config.overscanBiasJumpLocation)
2100 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2101 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
2102 self.config.overscanBiasJumpDevices)):
2103 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
2104 yLower = self.config.overscanBiasJumpLocation
2105 yUpper = dataBBox.getHeight() - yLower
2106 else:
2107 yUpper = self.config.overscanBiasJumpLocation
2108 yLower = dataBBox.getHeight() - yUpper
2109
2110 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2111 lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
2112 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2113 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2114 yLower)))
2115
2116 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
2117 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
2118 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
2119 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2120 yUpper)))
2121 else:
2122 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2123 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
2124 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2125 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2126 oscanBBox.getHeight())))
2127
2128 # Perform overscan correction on subregions, ensuring saturated
2129 # pixels are masked.
2130 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
2131 ampImage = ccdExposure.maskedImage[imageBBox]
2132 overscanImage = ccdExposure.maskedImage[overscanBBox]
2133
2134 overscanArray = overscanImage.image.array
2135 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2136 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2137 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
2138
2139 statControl = afwMath.StatisticsControl()
2140 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2141
2142 overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp)
2143
2144 # Measure average overscan levels and record them in the metadata.
2145 levelStat = afwMath.MEDIAN
2146 sigmaStat = afwMath.STDEVCLIP
2147
2148 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2149 self.config.qa.flatness.nIter)
2150 metadata = ccdExposure.getMetadata()
2151 ampNum = amp.getName()
2152 # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
2153 if isinstance(overscanResults.overscanFit, float):
2154 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2155 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2156 else:
2157 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2158 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2159 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2160
2161 return overscanResults
2162
2163 def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None):
2164 """Set the variance plane using the gain and read noise
2165
2166 The read noise is calculated from the ``overscanImage`` if the
2167 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2168 the value from the amplifier data is used.
2169
2170 Parameters
2171 ----------
2172 ampExposure : `lsst.afw.image.Exposure`
2173 Exposure to process.
2174 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2175 Amplifier detector data.
2176 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2177 Image of overscan, required only for empirical read noise.
2178 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
2179 PTC dataset containing the gains and read noise.
2180
2181
2182 Raises
2183 ------
2184 RuntimeError
2185 Raised if either ``usePtcGains`` of ``usePtcReadNoise``
2186 are ``True``, but ptcDataset is not provided.
2187
2188 Raised if ```doEmpiricalReadNoise`` is ``True`` but
2189 ``overscanImage`` is ``None``.
2190
2191 See also
2192 --------
2193 lsst.ip.isr.isrFunctions.updateVariance
2194 """
2195 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2196 if self.config.usePtcGains:
2197 if ptcDataset is None:
2198 raise RuntimeError("No ptcDataset provided to use PTC gains.")
2199 else:
2200 gain = ptcDataset.gain[amp.getName()]
2201 self.log.info("Using gain from Photon Transfer Curve.")
2202 else:
2203 gain = amp.getGain()
2204
2205 if math.isnan(gain):
2206 gain = 1.0
2207 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2208 elif gain <= 0:
2209 patchedGain = 1.0
2210 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.",
2211 amp.getName(), gain, patchedGain)
2212 gain = patchedGain
2213
2214 if self.config.doEmpiricalReadNoise and overscanImage is None:
2215 raise RuntimeError("Overscan is none for EmpiricalReadNoise.")
2216
2217 if self.config.doEmpiricalReadNoise and overscanImage is not None:
2218 stats = afwMath.StatisticsControl()
2219 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2220 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2221 self.log.info("Calculated empirical read noise for amp %s: %f.",
2222 amp.getName(), readNoise)
2223 elif self.config.usePtcReadNoise:
2224 if ptcDataset is None:
2225 raise RuntimeError("No ptcDataset provided to use PTC readnoise.")
2226 else:
2227 readNoise = ptcDataset.noise[amp.getName()]
2228 self.log.info("Using read noise from Photon Transfer Curve.")
2229 else:
2230 readNoise = amp.getReadNoise()
2231
2232 isrFunctions.updateVariance(
2233 maskedImage=ampExposure.getMaskedImage(),
2234 gain=gain,
2235 readNoise=readNoise,
2236 )
2237
2238 def maskNegativeVariance(self, exposure):
2239 """Identify and mask pixels with negative variance values.
2240
2241 Parameters
2242 ----------
2243 exposure : `lsst.afw.image.Exposure`
2244 Exposure to process.
2245
2246 See Also
2247 --------
2248 lsst.ip.isr.isrFunctions.updateVariance
2249 """
2250 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
2251 bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
2252 exposure.mask.array[bad] |= maskPlane
2253
2254 def darkCorrection(self, exposure, darkExposure, invert=False):
2255 """Apply dark correction in place.
2256
2257 Parameters
2258 ----------
2259 exposure : `lsst.afw.image.Exposure`
2260 Exposure to process.
2261 darkExposure : `lsst.afw.image.Exposure`
2262 Dark exposure of the same size as ``exposure``.
2263 invert : `Bool`, optional
2264 If True, re-add the dark to an already corrected image.
2265
2266 Raises
2267 ------
2268 RuntimeError
2269 Raised if either ``exposure`` or ``darkExposure`` do not
2270 have their dark time defined.
2271
2272 See Also
2273 --------
2274 lsst.ip.isr.isrFunctions.darkCorrection
2275 """
2276 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2277 if math.isnan(expScale):
2278 raise RuntimeError("Exposure darktime is NAN.")
2279 if darkExposure.getInfo().getVisitInfo() is not None \
2280 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2281 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2282 else:
2283 # DM-17444: darkExposure.getInfo.getVisitInfo() is None
2284 # so getDarkTime() does not exist.
2285 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2286 darkScale = 1.0
2287
2288 isrFunctions.darkCorrection(
2289 maskedImage=exposure.getMaskedImage(),
2290 darkMaskedImage=darkExposure.getMaskedImage(),
2291 expScale=expScale,
2292 darkScale=darkScale,
2293 invert=invert,
2294 trimToFit=self.config.doTrimToMatchCalib
2295 )
2296
2297 def doLinearize(self, detector):
2298 """Check if linearization is needed for the detector cameraGeom.
2299
2300 Checks config.doLinearize and the linearity type of the first
2301 amplifier.
2302
2303 Parameters
2304 ----------
2305 detector : `lsst.afw.cameraGeom.Detector`
2306 Detector to get linearity type from.
2307
2308 Returns
2309 -------
2310 doLinearize : `Bool`
2311 If True, linearization should be performed.
2312 """
2313 return self.config.doLinearize and \
2314 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2315
2316 def flatCorrection(self, exposure, flatExposure, invert=False):
2317 """Apply flat correction in place.
2318
2319 Parameters
2320 ----------
2321 exposure : `lsst.afw.image.Exposure`
2322 Exposure to process.
2323 flatExposure : `lsst.afw.image.Exposure`
2324 Flat exposure of the same size as ``exposure``.
2325 invert : `Bool`, optional
2326 If True, unflatten an already flattened image.
2327
2328 See Also
2329 --------
2330 lsst.ip.isr.isrFunctions.flatCorrection
2331 """
2332 isrFunctions.flatCorrection(
2333 maskedImage=exposure.getMaskedImage(),
2334 flatMaskedImage=flatExposure.getMaskedImage(),
2335 scalingType=self.config.flatScalingType,
2336 userScale=self.config.flatUserScale,
2337 invert=invert,
2338 trimToFit=self.config.doTrimToMatchCalib
2339 )
2340
2341 def saturationDetection(self, exposure, amp):
2342 """Detect and mask saturated pixels in config.saturatedMaskName.
2343
2344 Parameters
2345 ----------
2346 exposure : `lsst.afw.image.Exposure`
2347 Exposure to process. Only the amplifier DataSec is processed.
2349 Amplifier detector data.
2350
2351 See Also
2352 --------
2353 lsst.ip.isr.isrFunctions.makeThresholdMask
2354 """
2355 if not math.isnan(amp.getSaturation()):
2356 maskedImage = exposure.getMaskedImage()
2357 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2358 isrFunctions.makeThresholdMask(
2359 maskedImage=dataView,
2360 threshold=amp.getSaturation(),
2361 growFootprints=0,
2362 maskName=self.config.saturatedMaskName,
2363 )
2364
2365 def saturationInterpolation(self, exposure):
2366 """Interpolate over saturated pixels, in place.
2367
2368 This method should be called after `saturationDetection`, to
2369 ensure that the saturated pixels have been identified in the
2370 SAT mask. It should also be called after `assembleCcd`, since
2371 saturated regions may cross amplifier boundaries.
2372
2373 Parameters
2374 ----------
2375 exposure : `lsst.afw.image.Exposure`
2376 Exposure to process.
2377
2378 See Also
2379 --------
2380 lsst.ip.isr.isrTask.saturationDetection
2381 lsst.ip.isr.isrFunctions.interpolateFromMask
2382 """
2383 isrFunctions.interpolateFromMask(
2384 maskedImage=exposure.getMaskedImage(),
2385 fwhm=self.config.fwhm,
2386 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2387 maskNameList=list(self.config.saturatedMaskName),
2388 )
2389
2390 def suspectDetection(self, exposure, amp):
2391 """Detect and mask suspect pixels in config.suspectMaskName.
2392
2393 Parameters
2394 ----------
2395 exposure : `lsst.afw.image.Exposure`
2396 Exposure to process. Only the amplifier DataSec is processed.
2398 Amplifier detector data.
2399
2400 See Also
2401 --------
2402 lsst.ip.isr.isrFunctions.makeThresholdMask
2403
2404 Notes
2405 -----
2406 Suspect pixels are pixels whose value is greater than
2407 amp.getSuspectLevel(). This is intended to indicate pixels that may be
2408 affected by unknown systematics; for example if non-linearity
2409 corrections above a certain level are unstable then that would be a
2410 useful value for suspectLevel. A value of `nan` indicates that no such
2411 level exists and no pixels are to be masked as suspicious.
2412 """
2413 suspectLevel = amp.getSuspectLevel()
2414 if math.isnan(suspectLevel):
2415 return
2416
2417 maskedImage = exposure.getMaskedImage()
2418 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2419 isrFunctions.makeThresholdMask(
2420 maskedImage=dataView,
2421 threshold=suspectLevel,
2422 growFootprints=0,
2423 maskName=self.config.suspectMaskName,
2424 )
2425
2426 def maskDefect(self, exposure, defectBaseList):
2427 """Mask defects using mask plane "BAD", in place.
2428
2429 Parameters
2430 ----------
2431 exposure : `lsst.afw.image.Exposure`
2432 Exposure to process.
2433 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2435 List of defects to mask.
2436
2437 Notes
2438 -----
2439 Call this after CCD assembly, since defects may cross amplifier
2440 boundaries.
2441 """
2442 maskedImage = exposure.getMaskedImage()
2443 if not isinstance(defectBaseList, Defects):
2444 # Promotes DefectBase to Defect
2445 defectList = Defects(defectBaseList)
2446 else:
2447 defectList = defectBaseList
2448 defectList.maskPixels(maskedImage, maskName="BAD")
2449
2450 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2451 """Mask edge pixels with applicable mask plane.
2452
2453 Parameters
2454 ----------
2455 exposure : `lsst.afw.image.Exposure`
2456 Exposure to process.
2457 numEdgePixels : `int`, optional
2458 Number of edge pixels to mask.
2459 maskPlane : `str`, optional
2460 Mask plane name to use.
2461 level : `str`, optional
2462 Level at which to mask edges.
2463 """
2464 maskedImage = exposure.getMaskedImage()
2465 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2466
2467 if numEdgePixels > 0:
2468 if level == 'DETECTOR':
2469 boxes = [maskedImage.getBBox()]
2470 elif level == 'AMP':
2471 boxes = [amp.getBBox() for amp in exposure.getDetector()]
2472
2473 for box in boxes:
2474 # This makes a bbox numEdgeSuspect pixels smaller than the
2475 # image on each side
2476 subImage = maskedImage[box]
2477 box.grow(-numEdgePixels)
2478 # Mask pixels outside box
2479 SourceDetectionTask.setEdgeBits(
2480 subImage,
2481 box,
2482 maskBitMask)
2483
2484 def maskAndInterpolateDefects(self, exposure, defectBaseList):
2485 """Mask and interpolate defects using mask plane "BAD", in place.
2486
2487 Parameters
2488 ----------
2489 exposure : `lsst.afw.image.Exposure`
2490 Exposure to process.
2491 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2493 List of defects to mask and interpolate.
2494
2495 See Also
2496 --------
2497 lsst.ip.isr.isrTask.maskDefect
2498 """
2499 self.maskDefectmaskDefect(exposure, defectBaseList)
2500 self.maskEdgesmaskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2501 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2502 isrFunctions.interpolateFromMask(
2503 maskedImage=exposure.getMaskedImage(),
2504 fwhm=self.config.fwhm,
2505 growSaturatedFootprints=0,
2506 maskNameList=["BAD"],
2507 )
2508
2509 def maskNan(self, exposure):
2510 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2511
2512 Parameters
2513 ----------
2514 exposure : `lsst.afw.image.Exposure`
2515 Exposure to process.
2516
2517 Notes
2518 -----
2519 We mask over all non-finite values (NaN, inf), including those
2520 that are masked with other bits (because those may or may not be
2521 interpolated over later, and we want to remove all NaN/infs).
2522 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2523 preserve the historical name.
2524 """
2525 maskedImage = exposure.getMaskedImage()
2526
2527 # Find and mask NaNs
2528 maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2529 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2530 numNans = maskNans(maskedImage, maskVal)
2531 self.metadata.set("NUMNANS", numNans)
2532 if numNans > 0:
2533 self.log.warning("There were %d unmasked NaNs.", numNans)
2534
2535 def maskAndInterpolateNan(self, exposure):
2536 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2537 in place.
2538
2539 Parameters
2540 ----------
2541 exposure : `lsst.afw.image.Exposure`
2542 Exposure to process.
2543
2544 See Also
2545 --------
2546 lsst.ip.isr.isrTask.maskNan
2547 """
2548 self.maskNanmaskNan(exposure)
2549 isrFunctions.interpolateFromMask(
2550 maskedImage=exposure.getMaskedImage(),
2551 fwhm=self.config.fwhm,
2552 growSaturatedFootprints=0,
2553 maskNameList=["UNMASKEDNAN"],
2554 )
2555
2556 def measureBackground(self, exposure, IsrQaConfig=None):
2557 """Measure the image background in subgrids, for quality control.
2558
2559 Parameters
2560 ----------
2561 exposure : `lsst.afw.image.Exposure`
2562 Exposure to process.
2563 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2564 Configuration object containing parameters on which background
2565 statistics and subgrids to use.
2566 """
2567 if IsrQaConfig is not None:
2568 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2569 IsrQaConfig.flatness.nIter)
2570 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2571 statsControl.setAndMask(maskVal)
2572 maskedImage = exposure.getMaskedImage()
2573 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2574 skyLevel = stats.getValue(afwMath.MEDIAN)
2575 skySigma = stats.getValue(afwMath.STDEVCLIP)
2576 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2577 metadata = exposure.getMetadata()
2578 metadata.set('SKYLEVEL', skyLevel)
2579 metadata.set('SKYSIGMA', skySigma)
2580
2581 # calcluating flatlevel over the subgrids
2582 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2583 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2584 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2585 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2586 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2587 skyLevels = numpy.zeros((nX, nY))
2588
2589 for j in range(nY):
2590 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2591 for i in range(nX):
2592 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2593
2594 xLLC = xc - meshXHalf
2595 yLLC = yc - meshYHalf
2596 xURC = xc + meshXHalf - 1
2597 yURC = yc + meshYHalf - 1
2598
2599 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2600 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2601
2602 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2603
2604 good = numpy.where(numpy.isfinite(skyLevels))
2605 skyMedian = numpy.median(skyLevels[good])
2606 flatness = (skyLevels[good] - skyMedian) / skyMedian
2607 flatness_rms = numpy.std(flatness)
2608 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2609
2610 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2611 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2612 nX, nY, flatness_pp, flatness_rms)
2613
2614 metadata.set('FLATNESS_PP', float(flatness_pp))
2615 metadata.set('FLATNESS_RMS', float(flatness_rms))
2616 metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2617 metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2618 metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2619
2620 def roughZeroPoint(self, exposure):
2621 """Set an approximate magnitude zero point for the exposure.
2622
2623 Parameters
2624 ----------
2625 exposure : `lsst.afw.image.Exposure`
2626 Exposure to process.
2627 """
2628 filterLabel = exposure.getFilterLabel()
2629 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2630
2631 if physicalFilter in self.config.fluxMag0T1:
2632 fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2633 else:
2634 self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter)
2635 fluxMag0 = self.config.defaultFluxMag0T1
2636
2637 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2638 if not expTime > 0: # handle NaN as well as <= 0
2639 self.log.warning("Non-positive exposure time; skipping rough zero point.")
2640 return
2641
2642 self.log.info("Setting rough magnitude zero point for filter %s: %f",
2643 physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2644 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2645
2646 def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2647 """Set valid polygon as the intersection of fpPolygon and chip corners.
2648
2649 Parameters
2650 ----------
2651 ccdExposure : `lsst.afw.image.Exposure`
2652 Exposure to process.
2653 fpPolygon : `lsst.afw.geom.Polygon`
2654 Polygon in focal plane coordinates.
2655 """
2656 # Get ccd corners in focal plane coordinates
2657 ccd = ccdExposure.getDetector()
2658 fpCorners = ccd.getCorners(FOCAL_PLANE)
2659 ccdPolygon = Polygon(fpCorners)
2660
2661 # Get intersection of ccd corners with fpPolygon
2662 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2663
2664 # Transform back to pixel positions and build new polygon
2665 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2666 validPolygon = Polygon(ccdPoints)
2667 ccdExposure.getInfo().setValidPolygon(validPolygon)
2668
2669 @contextmanager
2670 def flatContext(self, exp, flat, dark=None):
2671 """Context manager that applies and removes flats and darks,
2672 if the task is configured to apply them.
2673
2674 Parameters
2675 ----------
2677 Exposure to process.
2679 Flat exposure the same size as ``exp``.
2680 dark : `lsst.afw.image.Exposure`, optional
2681 Dark exposure the same size as ``exp``.
2682
2683 Yields
2684 ------
2686 The flat and dark corrected exposure.
2687 """
2688 if self.config.doDark and dark is not None:
2689 self.darkCorrectiondarkCorrection(exp, dark)
2690 if self.config.doFlat:
2691 self.flatCorrectionflatCorrection(exp, flat)
2692 try:
2693 yield exp
2694 finally:
2695 if self.config.doFlat:
2696 self.flatCorrectionflatCorrection(exp, flat, invert=True)
2697 if self.config.doDark and dark is not None:
2698 self.darkCorrectiondarkCorrection(exp, dark, invert=True)
2699
2700 def debugView(self, exposure, stepname):
2701 """Utility function to examine ISR exposure at different stages.
2702
2703 Parameters
2704 ----------
2705 exposure : `lsst.afw.image.Exposure`
2706 Exposure to view.
2707 stepname : `str`
2708 State of processing to view.
2709 """
2710 frame = getDebugFrame(self._display, stepname)
2711 if frame:
2712 display = getDisplay(frame)
2713 display.scale('asinh', 'zscale')
2714 display.mtv(exposure)
2715 prompt = "Press Enter to continue [c]... "
2716 while True:
2717 ans = input(prompt).lower()
2718 if ans in ("", "c",):
2719 break
2720
2721
2722class FakeAmp(object):
2723 """A Detector-like object that supports returning gain and saturation level
2724
2725 This is used when the input exposure does not have a detector.
2726
2727 Parameters
2728 ----------
2729 exposure : `lsst.afw.image.Exposure`
2730 Exposure to generate a fake amplifier for.
2731 config : `lsst.ip.isr.isrTaskConfig`
2732 Configuration to apply to the fake amplifier.
2733 """
2734
2735 def __init__(self, exposure, config):
2736 self._bbox_bbox = exposure.getBBox(afwImage.LOCAL)
2737 self._RawHorizontalOverscanBBox_RawHorizontalOverscanBBox = lsst.geom.Box2I()
2738 self._gain_gain = config.gain
2739 self._readNoise_readNoise = config.readNoise
2740 self._saturation_saturation = config.saturation
2741
2742 def getBBox(self):
2743 return self._bbox_bbox
2744
2745 def getRawBBox(self):
2746 return self._bbox_bbox
2747
2749 return self._RawHorizontalOverscanBBox_RawHorizontalOverscanBBox
2750
2751 def getGain(self):
2752 return self._gain_gain
2753
2754 def getReadNoise(self):
2755 return self._readNoise_readNoise
2756
2757 def getSaturation(self):
2758 return self._saturation_saturation
2759
2761 return float("NaN")
2762
2763
2764class RunIsrConfig(pexConfig.Config):
2765 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2766
2767
2768class RunIsrTask(pipeBase.CmdLineTask):
2769 """Task to wrap the default IsrTask to allow it to be retargeted.
2770
2771 The standard IsrTask can be called directly from a command line
2772 program, but doing so removes the ability of the task to be
2773 retargeted. As most cameras override some set of the IsrTask
2774 methods, this would remove those data-specific methods in the
2775 output post-ISR images. This wrapping class fixes the issue,
2776 allowing identical post-ISR images to be generated by both the
2777 processCcd and isrTask code.
2778 """
2779 ConfigClass = RunIsrConfig
2780 _DefaultName = "runIsr"
2781
2782 def __init__(self, *args, **kwargs):
2783 super().__init__(*args, **kwargs)
2784 self.makeSubtask("isr")
2785
2786 def runDataRef(self, dataRef):
2787 """
2788 Parameters
2789 ----------
2790 dataRef : `lsst.daf.persistence.ButlerDataRef`
2791 data reference of the detector data to be processed
2792
2793 Returns
2794 -------
2795 result : `pipeBase.Struct`
2796 Result struct with component:
2797
2798 - exposure : `lsst.afw.image.Exposure`
2799 Post-ISR processed exposure.
2800 """
2801 return self.isr.runDataRef(dataRef)
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2748
def __init__(self, exposure, config)
Definition: isrTask.py:2735
def __init__(self, *config=None)
Definition: isrTask.py:271
def flatCorrection(self, exposure, flatExposure, invert=False)
Definition: isrTask.py:2316
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2535
def saturationInterpolation(self, exposure)
Definition: isrTask.py:2365
def runDataRef(self, sensorRef)
Definition: isrTask.py:1770
def maskNan(self, exposure)
Definition: isrTask.py:2509
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1958
def debugView(self, exposure, stepname)
Definition: isrTask.py:2700
def ensureExposure(self, inputExp, camera=None, detectorNum=None)
Definition: isrTask.py:1869
def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True)
Definition: isrTask.py:1821
def maskNegativeVariance(self, exposure)
Definition: isrTask.py:2238
def saturationDetection(self, exposure, amp)
Definition: isrTask.py:2341
def maskDefect(self, exposure, defectBaseList)
Definition: isrTask.py:2426
def __init__(self, **kwargs)
Definition: isrTask.py:973
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:984
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
Definition: isrTask.py:2450
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:2033
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2556
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2620
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2484
def setValidPolygonIntersect(self, ccdExposure, fpPolygon)
Definition: isrTask.py:2646
def readIsrData(self, dataRef, rawExposure)
Definition: isrTask.py:1083
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, isGen3=False)
Definition: isrTask.py:1270
def doLinearize(self, detector)
Definition: isrTask.py:2297
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2670
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1921
def suspectDetection(self, exposure, amp)
Definition: isrTask.py:2390
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
Definition: isrTask.py:2163
def darkCorrection(self, exposure, darkExposure, invert=False)
Definition: isrTask.py:2254
def __init__(self, *args, **kwargs)
Definition: isrTask.py:2782
def runDataRef(self, dataRef)
Definition: isrTask.py:2786
def checkFilter(exposure, filterList, log)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
Definition: isrTask.py:63
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35