lsst.ip.isr g1eb1de935c+765d039e86
isrTask.py
Go to the documentation of this file.
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import math
23import numpy
24
25import lsst.geom
26import lsst.afw.image as afwImage
27import lsst.afw.math as afwMath
28import lsst.pex.config as pexConfig
29import lsst.pipe.base as pipeBase
30import lsst.pipe.base.connectionTypes as cT
31
32from contextlib import contextmanager
33from lsstDebug import getDebugFrame
34
35from lsst.afw.cameraGeom import NullLinearityType
36from lsst.afw.display import getDisplay
37from lsst.meas.algorithms.detection import SourceDetectionTask
38from lsst.utils.timer import timeMethod
39
40from . import isrFunctions
41from . import isrQa
42from . import linearize
43from .defects import Defects
44
45from .assembleCcdTask import AssembleCcdTask
46from .crosstalk import CrosstalkTask, CrosstalkCalib
47from .fringe import FringeTask
48from .isr import maskNans
49from .masking import MaskingTask
50from .overscan import OverscanCorrectionTask
51from .straylight import StrayLightTask
52from .vignette import VignetteTask
53from .ampOffset import AmpOffsetTask
54from .deferredCharge import DeferredChargeTask
55from .isrStatistics import IsrStatisticsTask
56from lsst.daf.butler import DimensionGraph
57
58
59__all__ = ["IsrTask", "IsrTaskConfig"]
60
61
62def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections):
63 """Lookup function to identify crosstalkSource entries.
64
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
67 populated.
68
69 Parameters
70 ----------
71 datasetType : `str`
72 Dataset to lookup.
73 registry : `lsst.daf.butler.Registry`
74 Butler registry to query.
75 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
76 Data id to transform to identify crosstalkSources. The
77 ``detector`` entry will be stripped.
78 collections : `lsst.daf.butler.CollectionSearch`
79 Collections to search through.
80
81 Returns
82 -------
83 results : `list` [`lsst.daf.butler.DatasetRef`]
84 List of datasets that match the query that will be used as
85 crosstalkSources.
86 """
87 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"]))
88 results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId,
89 findFirst=True))
90 # In some contexts, calling `.expanded()` to expand all data IDs in the
91 # query results can be a lot faster because it vectorizes lookups. But in
92 # this case, expandDataId shouldn't need to hit the database at all in the
93 # steady state, because only the detector record is unknown and those are
94 # cached in the registry.
95 return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records)) for ref in results]
96
97
98class IsrTaskConnections(pipeBase.PipelineTaskConnections,
99 dimensions={"instrument", "exposure", "detector"},
100 defaultTemplates={}):
101 ccdExposure = cT.Input(
102 name="raw",
103 doc="Input exposure to process.",
104 storageClass="Exposure",
105 dimensions=["instrument", "exposure", "detector"],
106 )
107 camera = cT.PrerequisiteInput(
108 name="camera",
109 storageClass="Camera",
110 doc="Input camera to construct complete exposures.",
111 dimensions=["instrument"],
112 isCalibration=True,
113 )
114
115 crosstalk = cT.PrerequisiteInput(
116 name="crosstalk",
117 doc="Input crosstalk object",
118 storageClass="CrosstalkCalib",
119 dimensions=["instrument", "detector"],
120 isCalibration=True,
121 minimum=0, # can fall back to cameraGeom
122 )
123 crosstalkSources = cT.PrerequisiteInput(
124 name="isrOverscanCorrected",
125 doc="Overscan corrected input images.",
126 storageClass="Exposure",
127 dimensions=["instrument", "exposure", "detector"],
128 deferLoad=True,
129 multiple=True,
130 lookupFunction=crosstalkSourceLookup,
131 minimum=0, # not needed for all instruments, no config to control this
132 )
133 bias = cT.PrerequisiteInput(
134 name="bias",
135 doc="Input bias calibration.",
136 storageClass="ExposureF",
137 dimensions=["instrument", "detector"],
138 isCalibration=True,
139 )
140 dark = cT.PrerequisiteInput(
141 name='dark',
142 doc="Input dark calibration.",
143 storageClass="ExposureF",
144 dimensions=["instrument", "detector"],
145 isCalibration=True,
146 )
147 flat = cT.PrerequisiteInput(
148 name="flat",
149 doc="Input flat calibration.",
150 storageClass="ExposureF",
151 dimensions=["instrument", "physical_filter", "detector"],
152 isCalibration=True,
153 )
154 ptc = cT.PrerequisiteInput(
155 name="ptc",
156 doc="Input Photon Transfer Curve dataset",
157 storageClass="PhotonTransferCurveDataset",
158 dimensions=["instrument", "detector"],
159 isCalibration=True,
160 )
161 fringes = cT.PrerequisiteInput(
162 name="fringe",
163 doc="Input fringe calibration.",
164 storageClass="ExposureF",
165 dimensions=["instrument", "physical_filter", "detector"],
166 isCalibration=True,
167 minimum=0, # only needed for some bands, even when enabled
168 )
169 strayLightData = cT.PrerequisiteInput(
170 name='yBackground',
171 doc="Input stray light calibration.",
172 storageClass="StrayLightData",
173 dimensions=["instrument", "physical_filter", "detector"],
174 deferLoad=True,
175 isCalibration=True,
176 minimum=0, # only needed for some bands, even when enabled
177 )
178 bfKernel = cT.PrerequisiteInput(
179 name='bfKernel',
180 doc="Input brighter-fatter kernel.",
181 storageClass="NumpyArray",
182 dimensions=["instrument"],
183 isCalibration=True,
184 minimum=0, # can use either bfKernel or newBFKernel
185 )
186 newBFKernel = cT.PrerequisiteInput(
187 name='brighterFatterKernel',
188 doc="Newer complete kernel + gain solutions.",
189 storageClass="BrighterFatterKernel",
190 dimensions=["instrument", "detector"],
191 isCalibration=True,
192 minimum=0, # can use either bfKernel or newBFKernel
193 )
194 defects = cT.PrerequisiteInput(
195 name='defects',
196 doc="Input defect tables.",
197 storageClass="Defects",
198 dimensions=["instrument", "detector"],
199 isCalibration=True,
200 )
201 linearizer = cT.PrerequisiteInput(
202 name='linearizer',
203 storageClass="Linearizer",
204 doc="Linearity correction calibration.",
205 dimensions=["instrument", "detector"],
206 isCalibration=True,
207 minimum=0, # can fall back to cameraGeom
208 )
209 opticsTransmission = cT.PrerequisiteInput(
210 name="transmission_optics",
211 storageClass="TransmissionCurve",
212 doc="Transmission curve due to the optics.",
213 dimensions=["instrument"],
214 isCalibration=True,
215 )
216 filterTransmission = cT.PrerequisiteInput(
217 name="transmission_filter",
218 storageClass="TransmissionCurve",
219 doc="Transmission curve due to the filter.",
220 dimensions=["instrument", "physical_filter"],
221 isCalibration=True,
222 )
223 sensorTransmission = cT.PrerequisiteInput(
224 name="transmission_sensor",
225 storageClass="TransmissionCurve",
226 doc="Transmission curve due to the sensor.",
227 dimensions=["instrument", "detector"],
228 isCalibration=True,
229 )
230 atmosphereTransmission = cT.PrerequisiteInput(
231 name="transmission_atmosphere",
232 storageClass="TransmissionCurve",
233 doc="Transmission curve due to the atmosphere.",
234 dimensions=["instrument"],
235 isCalibration=True,
236 )
237 illumMaskedImage = cT.PrerequisiteInput(
238 name="illum",
239 doc="Input illumination correction.",
240 storageClass="MaskedImageF",
241 dimensions=["instrument", "physical_filter", "detector"],
242 isCalibration=True,
243 )
244 deferredChargeCalib = cT.PrerequisiteInput(
245 name="deferredCharge",
246 doc="Deferred charge/CTI correction dataset.",
247 storageClass="IsrCalib",
248 dimensions=["instrument", "detector"],
249 isCalibration=True,
250 )
251
252 outputExposure = cT.Output(
253 name='postISRCCD',
254 doc="Output ISR processed exposure.",
255 storageClass="Exposure",
256 dimensions=["instrument", "exposure", "detector"],
257 )
258 preInterpExposure = cT.Output(
259 name='preInterpISRCCD',
260 doc="Output ISR processed exposure, with pixels left uninterpolated.",
261 storageClass="ExposureF",
262 dimensions=["instrument", "exposure", "detector"],
263 )
264 outputOssThumbnail = cT.Output(
265 name="OssThumb",
266 doc="Output Overscan-subtracted thumbnail image.",
267 storageClass="Thumbnail",
268 dimensions=["instrument", "exposure", "detector"],
269 )
270 outputFlattenedThumbnail = cT.Output(
271 name="FlattenedThumb",
272 doc="Output flat-corrected thumbnail image.",
273 storageClass="Thumbnail",
274 dimensions=["instrument", "exposure", "detector"],
275 )
276 outputStatistics = cT.Output(
277 name="isrStatistics",
278 doc="Output of additional statistics table.",
279 storageClass="StructuredDataDict",
280 dimensions=["instrument", "exposure", "detector"],
281 )
282
283 def __init__(self, *, config=None):
284 super().__init__(config=config)
285
286 if config.doBias is not True:
287 self.prerequisiteInputs.remove("bias")
288 if config.doLinearize is not True:
289 self.prerequisiteInputs.remove("linearizer")
290 if config.doCrosstalk is not True:
291 self.prerequisiteInputs.remove("crosstalkSources")
292 self.prerequisiteInputs.remove("crosstalk")
293 if config.doBrighterFatter is not True:
294 self.prerequisiteInputs.remove("bfKernel")
295 self.prerequisiteInputs.remove("newBFKernel")
296 if config.doDefect is not True:
297 self.prerequisiteInputs.remove("defects")
298 if config.doDark is not True:
299 self.prerequisiteInputs.remove("dark")
300 if config.doFlat is not True:
301 self.prerequisiteInputs.remove("flat")
302 if config.doFringe is not True:
303 self.prerequisiteInputs.remove("fringes")
304 if config.doStrayLight is not True:
305 self.prerequisiteInputs.remove("strayLightData")
306 if config.usePtcGains is not True and config.usePtcReadNoise is not True:
307 self.prerequisiteInputs.remove("ptc")
308 if config.doAttachTransmissionCurve is not True:
309 self.prerequisiteInputs.remove("opticsTransmission")
310 self.prerequisiteInputs.remove("filterTransmission")
311 self.prerequisiteInputs.remove("sensorTransmission")
312 self.prerequisiteInputs.remove("atmosphereTransmission")
313 else:
314 if config.doUseOpticsTransmission is not True:
315 self.prerequisiteInputs.remove("opticsTransmission")
316 if config.doUseFilterTransmission is not True:
317 self.prerequisiteInputs.remove("filterTransmission")
318 if config.doUseSensorTransmission is not True:
319 self.prerequisiteInputs.remove("sensorTransmission")
320 if config.doUseAtmosphereTransmission is not True:
321 self.prerequisiteInputs.remove("atmosphereTransmission")
322 if config.doIlluminationCorrection is not True:
323 self.prerequisiteInputs.remove("illumMaskedImage")
324 if config.doDeferredCharge is not True:
325 self.prerequisiteInputs.remove("deferredChargeCalib")
326
327 if config.doWrite is not True:
328 self.outputs.remove("outputExposure")
329 self.outputs.remove("preInterpExposure")
330 self.outputs.remove("outputFlattenedThumbnail")
331 self.outputs.remove("outputOssThumbnail")
332 self.outputs.remove("outputStatistics")
333
334 if config.doSaveInterpPixels is not True:
335 self.outputs.remove("preInterpExposure")
336 if config.qa.doThumbnailOss is not True:
337 self.outputs.remove("outputOssThumbnail")
338 if config.qa.doThumbnailFlattened is not True:
339 self.outputs.remove("outputFlattenedThumbnail")
340 if config.doCalculateStatistics is not True:
341 self.outputs.remove("outputStatistics")
342
343
344class IsrTaskConfig(pipeBase.PipelineTaskConfig,
345 pipelineConnections=IsrTaskConnections):
346 """Configuration parameters for IsrTask.
347
348 Items are grouped in the order in which they are executed by the task.
349 """
350 datasetType = pexConfig.Field(
351 dtype=str,
352 doc="Dataset type for input data; users will typically leave this alone, "
353 "but camera-specific ISR tasks will override it",
354 default="raw",
355 )
356
357 fallbackFilterName = pexConfig.Field(
358 dtype=str,
359 doc="Fallback default filter name for calibrations.",
360 optional=True
361 )
362 useFallbackDate = pexConfig.Field(
363 dtype=bool,
364 doc="Pass observation date when using fallback filter.",
365 default=False,
366 )
367 expectWcs = pexConfig.Field(
368 dtype=bool,
369 default=True,
370 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
371 )
372 fwhm = pexConfig.Field(
373 dtype=float,
374 doc="FWHM of PSF in arcseconds.",
375 default=1.0,
376 )
377 qa = pexConfig.ConfigField(
378 dtype=isrQa.IsrQaConfig,
379 doc="QA related configuration options.",
380 )
381
382 # Image conversion configuration
383 doConvertIntToFloat = pexConfig.Field(
384 dtype=bool,
385 doc="Convert integer raw images to floating point values?",
386 default=True,
387 )
388
389 # Saturated pixel handling.
390 doSaturation = pexConfig.Field(
391 dtype=bool,
392 doc="Mask saturated pixels? NB: this is totally independent of the"
393 " interpolation option - this is ONLY setting the bits in the mask."
394 " To have them interpolated make sure doSaturationInterpolation=True",
395 default=True,
396 )
397 saturatedMaskName = pexConfig.Field(
398 dtype=str,
399 doc="Name of mask plane to use in saturation detection and interpolation",
400 default="SAT",
401 )
402 saturation = pexConfig.Field(
403 dtype=float,
404 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
405 default=float("NaN"),
406 )
407 growSaturationFootprintSize = pexConfig.Field(
408 dtype=int,
409 doc="Number of pixels by which to grow the saturation footprints",
410 default=1,
411 )
412
413 # Suspect pixel handling.
414 doSuspect = pexConfig.Field(
415 dtype=bool,
416 doc="Mask suspect pixels?",
417 default=False,
418 )
419 suspectMaskName = pexConfig.Field(
420 dtype=str,
421 doc="Name of mask plane to use for suspect pixels",
422 default="SUSPECT",
423 )
424 numEdgeSuspect = pexConfig.Field(
425 dtype=int,
426 doc="Number of edge pixels to be flagged as untrustworthy.",
427 default=0,
428 )
429 edgeMaskLevel = pexConfig.ChoiceField(
430 dtype=str,
431 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
432 default="DETECTOR",
433 allowed={
434 'DETECTOR': 'Mask only the edges of the full detector.',
435 'AMP': 'Mask edges of each amplifier.',
436 },
437 )
438
439 # Initial masking options.
440 doSetBadRegions = pexConfig.Field(
441 dtype=bool,
442 doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
443 default=True,
444 )
445 badStatistic = pexConfig.ChoiceField(
446 dtype=str,
447 doc="How to estimate the average value for BAD regions.",
448 default='MEANCLIP',
449 allowed={
450 "MEANCLIP": "Correct using the (clipped) mean of good data",
451 "MEDIAN": "Correct using the median of the good data",
452 },
453 )
454
455 # Overscan subtraction configuration.
456 doOverscan = pexConfig.Field(
457 dtype=bool,
458 doc="Do overscan subtraction?",
459 default=True,
460 )
461 overscan = pexConfig.ConfigurableField(
462 target=OverscanCorrectionTask,
463 doc="Overscan subtraction task for image segments.",
464 )
465
466 # Amplifier to CCD assembly configuration
467 doAssembleCcd = pexConfig.Field(
468 dtype=bool,
469 default=True,
470 doc="Assemble amp-level exposures into a ccd-level exposure?"
471 )
472 assembleCcd = pexConfig.ConfigurableField(
473 target=AssembleCcdTask,
474 doc="CCD assembly task",
475 )
476
477 # General calibration configuration.
478 doAssembleIsrExposures = pexConfig.Field(
479 dtype=bool,
480 default=False,
481 doc="Assemble amp-level calibration exposures into ccd-level exposure?"
482 )
483 doTrimToMatchCalib = pexConfig.Field(
484 dtype=bool,
485 default=False,
486 doc="Trim raw data to match calibration bounding boxes?"
487 )
488
489 # Bias subtraction.
490 doBias = pexConfig.Field(
491 dtype=bool,
492 doc="Apply bias frame correction?",
493 default=True,
494 )
495 biasDataProductName = pexConfig.Field(
496 dtype=str,
497 doc="Name of the bias data product",
498 default="bias",
499 )
500 doBiasBeforeOverscan = pexConfig.Field(
501 dtype=bool,
502 doc="Reverse order of overscan and bias correction.",
503 default=False
504 )
505
506 # Deferred charge correction.
507 doDeferredCharge = pexConfig.Field(
508 dtype=bool,
509 doc="Apply deferred charge correction?",
510 default=False,
511 )
512 deferredChargeCorrection = pexConfig.ConfigurableField(
513 target=DeferredChargeTask,
514 doc="Deferred charge correction task.",
515 )
516
517 # Variance construction
518 doVariance = pexConfig.Field(
519 dtype=bool,
520 doc="Calculate variance?",
521 default=True
522 )
523 gain = pexConfig.Field(
524 dtype=float,
525 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
526 default=float("NaN"),
527 )
528 readNoise = pexConfig.Field(
529 dtype=float,
530 doc="The read noise to use if no Detector is present in the Exposure",
531 default=0.0,
532 )
533 doEmpiricalReadNoise = pexConfig.Field(
534 dtype=bool,
535 default=False,
536 doc="Calculate empirical read noise instead of value from AmpInfo data?"
537 )
538 usePtcReadNoise = pexConfig.Field(
539 dtype=bool,
540 default=False,
541 doc="Use readnoise values from the Photon Transfer Curve?"
542 )
543 maskNegativeVariance = pexConfig.Field(
544 dtype=bool,
545 default=True,
546 doc="Mask pixels that claim a negative variance? This likely indicates a failure "
547 "in the measurement of the overscan at an edge due to the data falling off faster "
548 "than the overscan model can account for it."
549 )
550 negativeVarianceMaskName = pexConfig.Field(
551 dtype=str,
552 default="BAD",
553 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
554 )
555 # Linearization.
556 doLinearize = pexConfig.Field(
557 dtype=bool,
558 doc="Correct for nonlinearity of the detector's response?",
559 default=True,
560 )
561
562 # Crosstalk.
563 doCrosstalk = pexConfig.Field(
564 dtype=bool,
565 doc="Apply intra-CCD crosstalk correction?",
566 default=False,
567 )
568 doCrosstalkBeforeAssemble = pexConfig.Field(
569 dtype=bool,
570 doc="Apply crosstalk correction before CCD assembly, and before trimming?",
571 default=False,
572 )
573 crosstalk = pexConfig.ConfigurableField(
574 target=CrosstalkTask,
575 doc="Intra-CCD crosstalk correction",
576 )
577
578 # Masking options.
579 doDefect = pexConfig.Field(
580 dtype=bool,
581 doc="Apply correction for CCD defects, e.g. hot pixels?",
582 default=True,
583 )
584 doNanMasking = pexConfig.Field(
585 dtype=bool,
586 doc="Mask non-finite (NAN, inf) pixels?",
587 default=True,
588 )
589 doWidenSaturationTrails = pexConfig.Field(
590 dtype=bool,
591 doc="Widen bleed trails based on their width?",
592 default=True
593 )
594
595 # Brighter-Fatter correction.
596 doBrighterFatter = pexConfig.Field(
597 dtype=bool,
598 default=False,
599 doc="Apply the brighter-fatter correction?"
600 )
601 brighterFatterLevel = pexConfig.ChoiceField(
602 dtype=str,
603 default="DETECTOR",
604 doc="The level at which to correct for brighter-fatter.",
605 allowed={
606 "AMP": "Every amplifier treated separately.",
607 "DETECTOR": "One kernel per detector",
608 }
609 )
610 brighterFatterMaxIter = pexConfig.Field(
611 dtype=int,
612 default=10,
613 doc="Maximum number of iterations for the brighter-fatter correction"
614 )
615 brighterFatterThreshold = pexConfig.Field(
616 dtype=float,
617 default=1000,
618 doc="Threshold used to stop iterating the brighter-fatter correction. It is the "
619 "absolute value of the difference between the current corrected image and the one "
620 "from the previous iteration summed over all the pixels."
621 )
622 brighterFatterApplyGain = pexConfig.Field(
623 dtype=bool,
624 default=True,
625 doc="Should the gain be applied when applying the brighter-fatter correction?"
626 )
627 brighterFatterMaskListToInterpolate = pexConfig.ListField(
628 dtype=str,
629 doc="List of mask planes that should be interpolated over when applying the brighter-fatter "
630 "correction.",
631 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"],
632 )
633 brighterFatterMaskGrowSize = pexConfig.Field(
634 dtype=int,
635 default=0,
636 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
637 "when brighter-fatter correction is applied."
638 )
639
640 # Dark subtraction.
641 doDark = pexConfig.Field(
642 dtype=bool,
643 doc="Apply dark frame correction?",
644 default=True,
645 )
646 darkDataProductName = pexConfig.Field(
647 dtype=str,
648 doc="Name of the dark data product",
649 default="dark",
650 )
651
652 # Camera-specific stray light removal.
653 doStrayLight = pexConfig.Field(
654 dtype=bool,
655 doc="Subtract stray light in the y-band (due to encoder LEDs)?",
656 default=False,
657 )
658 strayLight = pexConfig.ConfigurableField(
659 target=StrayLightTask,
660 doc="y-band stray light correction"
661 )
662
663 # Flat correction.
664 doFlat = pexConfig.Field(
665 dtype=bool,
666 doc="Apply flat field correction?",
667 default=True,
668 )
669 flatDataProductName = pexConfig.Field(
670 dtype=str,
671 doc="Name of the flat data product",
672 default="flat",
673 )
674 flatScalingType = pexConfig.ChoiceField(
675 dtype=str,
676 doc="The method for scaling the flat on the fly.",
677 default='USER',
678 allowed={
679 "USER": "Scale by flatUserScale",
680 "MEAN": "Scale by the inverse of the mean",
681 "MEDIAN": "Scale by the inverse of the median",
682 },
683 )
684 flatUserScale = pexConfig.Field(
685 dtype=float,
686 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
687 default=1.0,
688 )
689 doTweakFlat = pexConfig.Field(
690 dtype=bool,
691 doc="Tweak flats to match observed amplifier ratios?",
692 default=False
693 )
694
695 # Amplifier normalization based on gains instead of using flats
696 # configuration.
697 doApplyGains = pexConfig.Field(
698 dtype=bool,
699 doc="Correct the amplifiers for their gains instead of applying flat correction",
700 default=False,
701 )
702 usePtcGains = pexConfig.Field(
703 dtype=bool,
704 doc="Use the gain values from the Photon Transfer Curve?",
705 default=False,
706 )
707 normalizeGains = pexConfig.Field(
708 dtype=bool,
709 doc="Normalize all the amplifiers in each CCD to have the same median value.",
710 default=False,
711 )
712
713 # Fringe correction.
714 doFringe = pexConfig.Field(
715 dtype=bool,
716 doc="Apply fringe correction?",
717 default=True,
718 )
719 fringe = pexConfig.ConfigurableField(
720 target=FringeTask,
721 doc="Fringe subtraction task",
722 )
723 fringeAfterFlat = pexConfig.Field(
724 dtype=bool,
725 doc="Do fringe subtraction after flat-fielding?",
726 default=True,
727 )
728
729 # Amp offset correction.
730 doAmpOffset = pexConfig.Field(
731 doc="Calculate and apply amp offset corrections?",
732 dtype=bool,
733 default=False,
734 )
735 ampOffset = pexConfig.ConfigurableField(
736 doc="Amp offset correction task.",
737 target=AmpOffsetTask,
738 )
739
740 # Initial CCD-level background statistics options.
741 doMeasureBackground = pexConfig.Field(
742 dtype=bool,
743 doc="Measure the background level on the reduced image?",
744 default=False,
745 )
746
747 # Camera-specific masking configuration.
748 doCameraSpecificMasking = pexConfig.Field(
749 dtype=bool,
750 doc="Mask camera-specific bad regions?",
751 default=False,
752 )
753 masking = pexConfig.ConfigurableField(
754 target=MaskingTask,
755 doc="Masking task."
756 )
757
758 # Interpolation options.
759 doInterpolate = pexConfig.Field(
760 dtype=bool,
761 doc="Interpolate masked pixels?",
762 default=True,
763 )
764 doSaturationInterpolation = pexConfig.Field(
765 dtype=bool,
766 doc="Perform interpolation over pixels masked as saturated?"
767 " NB: This is independent of doSaturation; if that is False this plane"
768 " will likely be blank, resulting in a no-op here.",
769 default=True,
770 )
771 doNanInterpolation = pexConfig.Field(
772 dtype=bool,
773 doc="Perform interpolation over pixels masked as NaN?"
774 " NB: This is independent of doNanMasking; if that is False this plane"
775 " will likely be blank, resulting in a no-op here.",
776 default=True,
777 )
778 doNanInterpAfterFlat = pexConfig.Field(
779 dtype=bool,
780 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
781 "also have to interpolate them before flat-fielding."),
782 default=False,
783 )
784 maskListToInterpolate = pexConfig.ListField(
785 dtype=str,
786 doc="List of mask planes that should be interpolated.",
787 default=['SAT', 'BAD'],
788 )
789 doSaveInterpPixels = pexConfig.Field(
790 dtype=bool,
791 doc="Save a copy of the pre-interpolated pixel values?",
792 default=False,
793 )
794
795 # Default photometric calibration options.
796 fluxMag0T1 = pexConfig.DictField(
797 keytype=str,
798 itemtype=float,
799 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
800 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
801 ))
802 )
803 defaultFluxMag0T1 = pexConfig.Field(
804 dtype=float,
805 doc="Default value for fluxMag0T1 (for an unrecognized filter).",
806 default=pow(10.0, 0.4*28.0)
807 )
808
809 # Vignette correction configuration.
810 doVignette = pexConfig.Field(
811 dtype=bool,
812 doc=("Compute and attach the validPolygon defining the unvignetted region to the exposure "
813 "according to vignetting parameters?"),
814 default=False,
815 )
816 doMaskVignettePolygon = pexConfig.Field(
817 dtype=bool,
818 doc=("Add a mask bit for pixels within the vignetted region. Ignored if doVignette "
819 "is False"),
820 default=True,
821 )
822 vignetteValue = pexConfig.Field(
823 dtype=float,
824 doc="Value to replace image array pixels with in the vignetted region? Ignored if None.",
825 optional=True,
826 default=None,
827 )
828 vignette = pexConfig.ConfigurableField(
829 target=VignetteTask,
830 doc="Vignetting task.",
831 )
832
833 # Transmission curve configuration.
834 doAttachTransmissionCurve = pexConfig.Field(
835 dtype=bool,
836 default=False,
837 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
838 )
839 doUseOpticsTransmission = pexConfig.Field(
840 dtype=bool,
841 default=True,
842 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
843 )
844 doUseFilterTransmission = pexConfig.Field(
845 dtype=bool,
846 default=True,
847 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
848 )
849 doUseSensorTransmission = pexConfig.Field(
850 dtype=bool,
851 default=True,
852 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
853 )
854 doUseAtmosphereTransmission = pexConfig.Field(
855 dtype=bool,
856 default=True,
857 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
858 )
859
860 # Illumination correction.
861 doIlluminationCorrection = pexConfig.Field(
862 dtype=bool,
863 default=False,
864 doc="Perform illumination correction?"
865 )
866 illuminationCorrectionDataProductName = pexConfig.Field(
867 dtype=str,
868 doc="Name of the illumination correction data product.",
869 default="illumcor",
870 )
871 illumScale = pexConfig.Field(
872 dtype=float,
873 doc="Scale factor for the illumination correction.",
874 default=1.0,
875 )
876 illumFilters = pexConfig.ListField(
877 dtype=str,
878 default=[],
879 doc="Only perform illumination correction for these filters."
880 )
881
882 # Calculate additional statistics?
883 doCalculateStatistics = pexConfig.Field(
884 dtype=bool,
885 doc="Should additional ISR statistics be calculated?",
886 default=False,
887 )
888 isrStats = pexConfig.ConfigurableField(
889 target=IsrStatisticsTask,
890 doc="Task to calculate additional statistics.",
891 )
892
893 # Write the outputs to disk. If ISR is run as a subtask, this may not
894 # be needed.
895 doWrite = pexConfig.Field(
896 dtype=bool,
897 doc="Persist postISRCCD?",
898 default=True,
899 )
900
901 def validate(self):
902 super().validate()
903 if self.doFlat and self.doApplyGains:
904 raise ValueError("You may not specify both doFlat and doApplyGains")
906 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
911 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
912 self.maskListToInterpolate.append("UNMASKEDNAN")
913
914
915class IsrTask(pipeBase.PipelineTask):
916 """Apply common instrument signature correction algorithms to a raw frame.
917
918 The process for correcting imaging data is very similar from
919 camera to camera. This task provides a vanilla implementation of
920 doing these corrections, including the ability to turn certain
921 corrections off if they are not needed. The inputs to the primary
922 method, `run()`, are a raw exposure to be corrected and the
923 calibration data products. The raw input is a single chip sized
924 mosaic of all amps including overscans and other non-science
925 pixels.
926
927 The __init__ method sets up the subtasks for ISR processing, using
928 the defaults from `lsst.ip.isr`.
929
930 Parameters
931 ----------
932 args : `list`
933 Positional arguments passed to the Task constructor.
934 None used at this time.
935 kwargs : `dict`, optional
936 Keyword arguments passed on to the Task constructor.
937 None used at this time.
938 """
939 ConfigClass = IsrTaskConfig
940 _DefaultName = "isr"
941
942 def __init__(self, **kwargs):
943 super().__init__(**kwargs)
944 self.makeSubtask("assembleCcd")
945 self.makeSubtask("crosstalk")
946 self.makeSubtask("strayLight")
947 self.makeSubtask("fringe")
948 self.makeSubtask("masking")
949 self.makeSubtask("overscan")
950 self.makeSubtask("vignette")
951 self.makeSubtask("ampOffset")
952 self.makeSubtask("deferredChargeCorrection")
953 self.makeSubtask("isrStats")
954
955 def runQuantum(self, butlerQC, inputRefs, outputRefs):
956 inputs = butlerQC.get(inputRefs)
957
958 try:
959 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
960 except Exception as e:
961 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
962 (inputRefs, e))
963
964 detector = inputs['ccdExposure'].getDetector()
965
966 if self.config.doCrosstalk is True:
967 # Crosstalk sources need to be defined by the pipeline
968 # yaml if they exist.
969 if 'crosstalk' in inputs and inputs['crosstalk'] is not None:
970 if not isinstance(inputs['crosstalk'], CrosstalkCalib):
971 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk'])
972 else:
973 coeffVector = (self.config.crosstalk.crosstalkValues
974 if self.config.crosstalk.useConfigCoefficients else None)
975 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
976 inputs['crosstalk'] = crosstalkCalib
977 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0:
978 if 'crosstalkSources' not in inputs:
979 self.log.warning("No crosstalkSources found for chip with interChip terms!")
980
981 if self.doLinearize(detector) is True:
982 if 'linearizer' in inputs:
983 if isinstance(inputs['linearizer'], dict):
984 linearizer = linearize.Linearizer(detector=detector, log=self.log)
985 linearizer.fromYaml(inputs['linearizer'])
986 self.log.warning("Dictionary linearizers will be deprecated in DM-28741.")
987 elif isinstance(inputs['linearizer'], numpy.ndarray):
988 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None),
989 detector=detector,
990 log=self.log)
991 self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.")
992 else:
993 linearizer = inputs['linearizer']
994 linearizer.log = self.log
995 inputs['linearizer'] = linearizer
996 else:
997 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log)
998 self.log.warning("Constructing linearizer from cameraGeom information.")
999
1000 if self.config.doDefect is True:
1001 if "defects" in inputs and inputs['defects'] is not None:
1002 # defects is loaded as a BaseCatalog with columns
1003 # x0, y0, width, height. Masking expects a list of defects
1004 # defined by their bounding box
1005 if not isinstance(inputs["defects"], Defects):
1006 inputs["defects"] = Defects.fromTable(inputs["defects"])
1007
1008 # Load the correct style of brighter-fatter kernel, and repack
1009 # the information as a numpy array.
1010 if self.config.doBrighterFatter:
1011 brighterFatterKernel = inputs.pop('newBFKernel', None)
1012 if brighterFatterKernel is None:
1013 brighterFatterKernel = inputs.get('bfKernel', None)
1014
1015 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1016 # This is a ISR calib kernel
1017 detName = detector.getName()
1018 level = brighterFatterKernel.level
1019
1020 # This is expected to be a dictionary of amp-wise gains.
1021 inputs['bfGains'] = brighterFatterKernel.gain
1022 if self.config.brighterFatterLevel == 'DETECTOR':
1023 if level == 'DETECTOR':
1024 if detName in brighterFatterKernel.detKernels:
1025 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1026 else:
1027 raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1028 elif level == 'AMP':
1029 self.log.warning("Making DETECTOR level kernel from AMP based brighter "
1030 "fatter kernels.")
1031 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1032 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1033 elif self.config.brighterFatterLevel == 'AMP':
1034 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1035
1036 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
1037 expId = inputs['ccdExposure'].info.id
1038 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
1039 expId=expId,
1040 assembler=self.assembleCcd
1041 if self.config.doAssembleIsrExposures else None)
1042 else:
1043 inputs['fringes'] = pipeBase.Struct(fringes=None)
1044
1045 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
1046 if 'strayLightData' not in inputs:
1047 inputs['strayLightData'] = None
1048
1049 outputs = self.run(**inputs)
1050 butlerQC.put(outputs, outputRefs)
1051
1052 @timeMethod
1053 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1054 crosstalk=None, crosstalkSources=None,
1055 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1056 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1057 sensorTransmission=None, atmosphereTransmission=None,
1058 detectorNum=None, strayLightData=None, illumMaskedImage=None,
1059 deferredCharge=None,
1060 ):
1061 """Perform instrument signature removal on an exposure.
1062
1063 Steps included in the ISR processing, in order performed, are:
1064 - saturation and suspect pixel masking
1065 - overscan subtraction
1066 - CCD assembly of individual amplifiers
1067 - bias subtraction
1068 - variance image construction
1069 - linearization of non-linear response
1070 - crosstalk masking
1071 - brighter-fatter correction
1072 - dark subtraction
1073 - fringe correction
1074 - stray light subtraction
1075 - flat correction
1076 - masking of known defects and camera specific features
1077 - vignette calculation
1078 - appending transmission curve and distortion model
1079
1080 Parameters
1081 ----------
1082 ccdExposure : `lsst.afw.image.Exposure`
1083 The raw exposure that is to be run through ISR. The
1084 exposure is modified by this method.
1085 camera : `lsst.afw.cameraGeom.Camera`, optional
1086 The camera geometry for this exposure. Required if
1087 one or more of ``ccdExposure``, ``bias``, ``dark``, or
1088 ``flat`` does not have an associated detector.
1089 bias : `lsst.afw.image.Exposure`, optional
1090 Bias calibration frame.
1091 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1092 Functor for linearization.
1093 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1094 Calibration for crosstalk.
1095 crosstalkSources : `list`, optional
1096 List of possible crosstalk sources.
1097 dark : `lsst.afw.image.Exposure`, optional
1098 Dark calibration frame.
1099 flat : `lsst.afw.image.Exposure`, optional
1100 Flat calibration frame.
1102 Photon transfer curve dataset, with, e.g., gains
1103 and read noise.
1104 bfKernel : `numpy.ndarray`, optional
1105 Brighter-fatter kernel.
1106 bfGains : `dict` of `float`, optional
1107 Gains used to override the detector's nominal gains for the
1108 brighter-fatter correction. A dict keyed by amplifier name for
1109 the detector in question.
1110 defects : `lsst.ip.isr.Defects`, optional
1111 List of defects.
1112 fringes : `lsst.pipe.base.Struct`, optional
1113 Struct containing the fringe correction data, with
1114 elements:
1115 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1116 - ``seed``: random seed derived from the ccdExposureId for random
1117 number generator (`uint32`)
1118 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1119 A ``TransmissionCurve`` that represents the throughput of the,
1120 optics, to be evaluated in focal-plane coordinates.
1121 filterTransmission : `lsst.afw.image.TransmissionCurve`
1122 A ``TransmissionCurve`` that represents the throughput of the
1123 filter itself, to be evaluated in focal-plane coordinates.
1124 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1125 A ``TransmissionCurve`` that represents the throughput of the
1126 sensor itself, to be evaluated in post-assembly trimmed detector
1127 coordinates.
1128 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1129 A ``TransmissionCurve`` that represents the throughput of the
1130 atmosphere, assumed to be spatially constant.
1131 detectorNum : `int`, optional
1132 The integer number for the detector to process.
1133 strayLightData : `object`, optional
1134 Opaque object containing calibration information for stray-light
1135 correction. If `None`, no correction will be performed.
1136 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1137 Illumination correction image.
1138
1139 Returns
1140 -------
1141 result : `lsst.pipe.base.Struct`
1142 Result struct with component:
1143 - ``exposure`` : `afw.image.Exposure`
1144 The fully ISR corrected exposure.
1145 - ``outputExposure`` : `afw.image.Exposure`
1146 An alias for `exposure`
1147 - ``ossThumb`` : `numpy.ndarray`
1148 Thumbnail image of the exposure after overscan subtraction.
1149 - ``flattenedThumb`` : `numpy.ndarray`
1150 Thumbnail image of the exposure after flat-field correction.
1151 - ``outputStatistics`` : ``
1152 Values of the additional statistics calculated.
1153
1154 Raises
1155 ------
1156 RuntimeError
1157 Raised if a configuration option is set to True, but the
1158 required calibration data has not been specified.
1159
1160 Notes
1161 -----
1162 The current processed exposure can be viewed by setting the
1163 appropriate lsstDebug entries in the `debug.display`
1164 dictionary. The names of these entries correspond to some of
1165 the IsrTaskConfig Boolean options, with the value denoting the
1166 frame to use. The exposure is shown inside the matching
1167 option check and after the processing of that step has
1168 finished. The steps with debug points are:
1169
1170 doAssembleCcd
1171 doBias
1172 doCrosstalk
1173 doBrighterFatter
1174 doDark
1175 doFringe
1176 doStrayLight
1177 doFlat
1178
1179 In addition, setting the "postISRCCD" entry displays the
1180 exposure after all ISR processing has finished.
1181
1182 """
1183
1184 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1185 bias = self.ensureExposure(bias, camera, detectorNum)
1186 dark = self.ensureExposure(dark, camera, detectorNum)
1187 flat = self.ensureExposure(flat, camera, detectorNum)
1188
1189 ccd = ccdExposure.getDetector()
1190 filterLabel = ccdExposure.getFilter()
1191 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1192
1193 if not ccd:
1194 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1195 ccd = [FakeAmp(ccdExposure, self.config)]
1196
1197 # Validate Input
1198 if self.config.doBias and bias is None:
1199 raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1200 if self.doLinearize(ccd) and linearizer is None:
1201 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1202 if self.config.doBrighterFatter and bfKernel is None:
1203 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1204 if self.config.doDark and dark is None:
1205 raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1206 if self.config.doFlat and flat is None:
1207 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1208 if self.config.doDefect and defects is None:
1209 raise RuntimeError("Must supply defects if config.doDefect=True.")
1210 if (self.config.doFringe and physicalFilter in self.fringe.config.filters
1211 and fringes.fringes is None):
1212 # The `fringes` object needs to be a pipeBase.Struct, as
1213 # we use it as a `dict` for the parameters of
1214 # `FringeTask.run()`. The `fringes.fringes` `list` may
1215 # not be `None` if `doFringe=True`. Otherwise, raise.
1216 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1217 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters
1218 and illumMaskedImage is None):
1219 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1220 if (self.config.doDeferredCharge and deferredCharge is None):
1221 raise RuntimeError("Must supply a deferred charge calibration if config.doDeferredCharge=True.")
1222
1223 # Begin ISR processing.
1224 if self.config.doConvertIntToFloat:
1225 self.log.info("Converting exposure to floating point values.")
1226 ccdExposure = self.convertIntToFloat(ccdExposure)
1227
1228 if self.config.doBias and self.config.doBiasBeforeOverscan:
1229 self.log.info("Applying bias correction.")
1230 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1231 trimToFit=self.config.doTrimToMatchCalib)
1232 self.debugView(ccdExposure, "doBias")
1233
1234 # Amplifier level processing.
1235 overscans = []
1236 for amp in ccd:
1237 # if ccdExposure is one amp,
1238 # check for coverage to prevent performing ops multiple times
1239 if ccdExposure.getBBox().contains(amp.getBBox()):
1240 # Check for fully masked bad amplifiers,
1241 # and generate masks for SUSPECT and SATURATED values.
1242 badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1243
1244 if self.config.doOverscan and not badAmp:
1245 # Overscan correction on amp-by-amp basis.
1246 overscanResults = self.overscanCorrection(ccdExposure, amp)
1247 self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1248 if overscanResults is not None and \
1249 self.config.qa is not None and self.config.qa.saveStats is True:
1250
1251 self.metadata[f"FIT MEDIAN {amp.getName()}"] = overscanResults.overscanMean
1252 self.metadata[f"FIT STDEV {amp.getName()}"] = overscanResults.overscanSigma
1253 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1254 amp.getName(), overscanResults.overscanMean,
1255 overscanResults.overscanSigma)
1256
1257 self.metadata[f"RESIDUAL MEDIAN {amp.getName()}"] = overscanResults.residualMean
1258 self.metadata[f"RESIDUAL STDEV {amp.getName()}"] = overscanResults.residualSigma
1259 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f",
1260 amp.getName(), overscanResults.residualMean,
1261 overscanResults.residualSigma)
1262
1263 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1264 else:
1265 if badAmp:
1266 self.log.warning("Amplifier %s is bad.", amp.getName())
1267 overscanResults = None
1268
1269 overscans.append(overscanResults if overscanResults is not None else None)
1270 else:
1271 self.log.info("Skipped OSCAN for %s.", amp.getName())
1272
1273 if self.config.doDeferredCharge:
1274 self.log.info("Applying deferred charge/CTI correction.")
1275 self.deferredChargeCorrection.run(ccdExposure, deferredCharge)
1276 self.debugView(ccdExposure, "doDeferredCharge")
1277
1278 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1279 self.log.info("Applying crosstalk correction.")
1280 self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1281 crosstalkSources=crosstalkSources, camera=camera)
1282 self.debugView(ccdExposure, "doCrosstalk")
1283
1284 if self.config.doAssembleCcd:
1285 self.log.info("Assembling CCD from amplifiers.")
1286 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1287
1288 if self.config.expectWcs and not ccdExposure.getWcs():
1289 self.log.warning("No WCS found in input exposure.")
1290 self.debugView(ccdExposure, "doAssembleCcd")
1291
1292 ossThumb = None
1293 if self.config.qa.doThumbnailOss:
1294 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1295
1296 if self.config.doBias and not self.config.doBiasBeforeOverscan:
1297 self.log.info("Applying bias correction.")
1298 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1299 trimToFit=self.config.doTrimToMatchCalib)
1300 self.debugView(ccdExposure, "doBias")
1301
1302 if self.config.doVariance:
1303 for amp, overscanResults in zip(ccd, overscans):
1304 if ccdExposure.getBBox().contains(amp.getBBox()):
1305 self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1306 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1307 if overscanResults is not None:
1308 self.updateVariance(ampExposure, amp,
1309 overscanImage=overscanResults.overscanImage,
1310 ptcDataset=ptc)
1311 else:
1312 self.updateVariance(ampExposure, amp,
1313 overscanImage=None,
1314 ptcDataset=ptc)
1315 if self.config.qa is not None and self.config.qa.saveStats is True:
1316 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1317 afwMath.MEDIAN | afwMath.STDEVCLIP)
1318 self.metadata[f"ISR VARIANCE {amp.getName()} MEDIAN"] = \
1319 qaStats.getValue(afwMath.MEDIAN)
1320 self.metadata[f"ISR VARIANCE {amp.getName()} STDEV"] = \
1321 qaStats.getValue(afwMath.STDEVCLIP)
1322 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1323 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1324 qaStats.getValue(afwMath.STDEVCLIP))
1325 if self.config.maskNegativeVariance:
1326 self.maskNegativeVariance(ccdExposure)
1327
1328 if self.doLinearize(ccd):
1329 self.log.info("Applying linearizer.")
1330 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1331 detector=ccd, log=self.log)
1332
1333 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1334 self.log.info("Applying crosstalk correction.")
1335 self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1336 crosstalkSources=crosstalkSources, isTrimmed=True)
1337 self.debugView(ccdExposure, "doCrosstalk")
1338
1339 # Masking block. Optionally mask known defects, NAN/inf pixels,
1340 # widen trails, and do anything else the camera needs. Saturated and
1341 # suspect pixels have already been masked.
1342 if self.config.doDefect:
1343 self.log.info("Masking defects.")
1344 self.maskDefect(ccdExposure, defects)
1345
1346 if self.config.numEdgeSuspect > 0:
1347 self.log.info("Masking edges as SUSPECT.")
1348 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1349 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
1350
1351 if self.config.doNanMasking:
1352 self.log.info("Masking non-finite (NAN, inf) value pixels.")
1353 self.maskNan(ccdExposure)
1354
1355 if self.config.doWidenSaturationTrails:
1356 self.log.info("Widening saturation trails.")
1357 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1358
1359 if self.config.doCameraSpecificMasking:
1360 self.log.info("Masking regions for camera specific reasons.")
1361 self.masking.run(ccdExposure)
1362
1363 if self.config.doBrighterFatter:
1364 # We need to apply flats and darks before we can interpolate, and
1365 # we need to interpolate before we do B-F, but we do B-F without
1366 # the flats and darks applied so we can work in units of electrons
1367 # or holes. This context manager applies and then removes the darks
1368 # and flats.
1369 #
1370 # We also do not want to interpolate values here, so operate on
1371 # temporary images so we can apply only the BF-correction and roll
1372 # back the interpolation.
1373 interpExp = ccdExposure.clone()
1374 with self.flatContext(interpExp, flat, dark):
1375 isrFunctions.interpolateFromMask(
1376 maskedImage=interpExp.getMaskedImage(),
1377 fwhm=self.config.fwhm,
1378 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1379 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1380 )
1381 bfExp = interpExp.clone()
1382
1383 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.",
1384 type(bfKernel), type(bfGains))
1385 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1386 self.config.brighterFatterMaxIter,
1387 self.config.brighterFatterThreshold,
1388 self.config.brighterFatterApplyGain,
1389 bfGains)
1390 if bfResults[1] == self.config.brighterFatterMaxIter:
1391 self.log.warning("Brighter-fatter correction did not converge, final difference %f.",
1392 bfResults[0])
1393 else:
1394 self.log.info("Finished brighter-fatter correction in %d iterations.",
1395 bfResults[1])
1396 image = ccdExposure.getMaskedImage().getImage()
1397 bfCorr = bfExp.getMaskedImage().getImage()
1398 bfCorr -= interpExp.getMaskedImage().getImage()
1399 image += bfCorr
1400
1401 # Applying the brighter-fatter correction applies a
1402 # convolution to the science image. At the edges this
1403 # convolution may not have sufficient valid pixels to
1404 # produce a valid correction. Mark pixels within the size
1405 # of the brighter-fatter kernel as EDGE to warn of this
1406 # fact.
1407 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1408 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1409 maskPlane="EDGE")
1410
1411 if self.config.brighterFatterMaskGrowSize > 0:
1412 self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1413 for maskPlane in self.config.brighterFatterMaskListToInterpolate:
1414 isrFunctions.growMasks(ccdExposure.getMask(),
1415 radius=self.config.brighterFatterMaskGrowSize,
1416 maskNameList=maskPlane,
1417 maskValue=maskPlane)
1418
1419 self.debugView(ccdExposure, "doBrighterFatter")
1420
1421 if self.config.doDark:
1422 self.log.info("Applying dark correction.")
1423 self.darkCorrection(ccdExposure, dark)
1424 self.debugView(ccdExposure, "doDark")
1425
1426 if self.config.doFringe and not self.config.fringeAfterFlat:
1427 self.log.info("Applying fringe correction before flat.")
1428 self.fringe.run(ccdExposure, **fringes.getDict())
1429 self.debugView(ccdExposure, "doFringe")
1430
1431 if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1432 self.log.info("Checking strayLight correction.")
1433 self.strayLight.run(ccdExposure, strayLightData)
1434 self.debugView(ccdExposure, "doStrayLight")
1435
1436 if self.config.doFlat:
1437 self.log.info("Applying flat correction.")
1438 self.flatCorrection(ccdExposure, flat)
1439 self.debugView(ccdExposure, "doFlat")
1440
1441 if self.config.doApplyGains:
1442 self.log.info("Applying gain correction instead of flat.")
1443 if self.config.usePtcGains:
1444 self.log.info("Using gains from the Photon Transfer Curve.")
1445 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1446 ptcGains=ptc.gain)
1447 else:
1448 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1449
1450 if self.config.doFringe and self.config.fringeAfterFlat:
1451 self.log.info("Applying fringe correction after flat.")
1452 self.fringe.run(ccdExposure, **fringes.getDict())
1453
1454 if self.config.doVignette:
1455 if self.config.doMaskVignettePolygon:
1456 self.log.info("Constructing, attaching, and masking vignette polygon.")
1457 else:
1458 self.log.info("Constructing and attaching vignette polygon.")
1459 self.vignettePolygon = self.vignette.run(
1460 exposure=ccdExposure, doUpdateMask=self.config.doMaskVignettePolygon,
1461 vignetteValue=self.config.vignetteValue, log=self.log)
1462
1463 if self.config.doAttachTransmissionCurve:
1464 self.log.info("Adding transmission curves.")
1465 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1466 filterTransmission=filterTransmission,
1467 sensorTransmission=sensorTransmission,
1468 atmosphereTransmission=atmosphereTransmission)
1469
1470 flattenedThumb = None
1471 if self.config.qa.doThumbnailFlattened:
1472 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1473
1474 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters:
1475 self.log.info("Performing illumination correction.")
1476 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1477 illumMaskedImage, illumScale=self.config.illumScale,
1478 trimToFit=self.config.doTrimToMatchCalib)
1479
1480 preInterpExp = None
1481 if self.config.doSaveInterpPixels:
1482 preInterpExp = ccdExposure.clone()
1483
1484 # Reset and interpolate bad pixels.
1485 #
1486 # Large contiguous bad regions (which should have the BAD mask
1487 # bit set) should have their values set to the image median.
1488 # This group should include defects and bad amplifiers. As the
1489 # area covered by these defects are large, there's little
1490 # reason to expect that interpolation would provide a more
1491 # useful value.
1492 #
1493 # Smaller defects can be safely interpolated after the larger
1494 # regions have had their pixel values reset. This ensures
1495 # that the remaining defects adjacent to bad amplifiers (as an
1496 # example) do not attempt to interpolate extreme values.
1497 if self.config.doSetBadRegions:
1498 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1499 if badPixelCount > 0:
1500 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1501
1502 if self.config.doInterpolate:
1503 self.log.info("Interpolating masked pixels.")
1504 isrFunctions.interpolateFromMask(
1505 maskedImage=ccdExposure.getMaskedImage(),
1506 fwhm=self.config.fwhm,
1507 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1508 maskNameList=list(self.config.maskListToInterpolate)
1509 )
1510
1511 self.roughZeroPoint(ccdExposure)
1512
1513 # correct for amp offsets within the CCD
1514 if self.config.doAmpOffset:
1515 self.log.info("Correcting amp offsets.")
1516 self.ampOffset.run(ccdExposure)
1517
1518 if self.config.doMeasureBackground:
1519 self.log.info("Measuring background level.")
1520 self.measureBackground(ccdExposure, self.config.qa)
1521
1522 if self.config.qa is not None and self.config.qa.saveStats is True:
1523 for amp in ccd:
1524 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1525 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1526 afwMath.MEDIAN | afwMath.STDEVCLIP)
1527 self.metadata[f"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN)
1528 self.metadata[f"ISR BACKGROUND {amp.getName()} STDEV"] = \
1529 qaStats.getValue(afwMath.STDEVCLIP)
1530 self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1531 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1532 qaStats.getValue(afwMath.STDEVCLIP))
1533
1534 # calculate additional statistics.
1535 outputStatistics = None
1536 if self.config.doCalculateStatistics:
1537 outputStatistics = self.isrStats.run(ccdExposure, overscanResults=overscans,
1538 ptc=ptc).results
1539
1540 self.debugView(ccdExposure, "postISRCCD")
1541
1542 return pipeBase.Struct(
1543 exposure=ccdExposure,
1544 ossThumb=ossThumb,
1545 flattenedThumb=flattenedThumb,
1546
1547 preInterpExposure=preInterpExp,
1548 outputExposure=ccdExposure,
1549 outputOssThumbnail=ossThumb,
1550 outputFlattenedThumbnail=flattenedThumb,
1551 outputStatistics=outputStatistics,
1552 )
1553
1554 def ensureExposure(self, inputExp, camera=None, detectorNum=None):
1555 """Ensure that the data returned by Butler is a fully constructed exp.
1556
1557 ISR requires exposure-level image data for historical reasons, so if we
1558 did not recieve that from Butler, construct it from what we have,
1559 modifying the input in place.
1560
1561 Parameters
1562 ----------
1563 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`,
1564 or `lsst.afw.image.ImageF`
1565 The input data structure obtained from Butler.
1566 camera : `lsst.afw.cameraGeom.camera`, optional
1567 The camera associated with the image. Used to find the appropriate
1568 detector if detector is not already set.
1569 detectorNum : `int`, optional
1570 The detector in the camera to attach, if the detector is not
1571 already set.
1572
1573 Returns
1574 -------
1575 inputExp : `lsst.afw.image.Exposure`
1576 The re-constructed exposure, with appropriate detector parameters.
1577
1578 Raises
1579 ------
1580 TypeError
1581 Raised if the input data cannot be used to construct an exposure.
1582 """
1583 if isinstance(inputExp, afwImage.DecoratedImageU):
1584 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1585 elif isinstance(inputExp, afwImage.ImageF):
1586 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1587 elif isinstance(inputExp, afwImage.MaskedImageF):
1588 inputExp = afwImage.makeExposure(inputExp)
1589 elif isinstance(inputExp, afwImage.Exposure):
1590 pass
1591 elif inputExp is None:
1592 # Assume this will be caught by the setup if it is a problem.
1593 return inputExp
1594 else:
1595 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1596 (type(inputExp), ))
1597
1598 if inputExp.getDetector() is None:
1599 if camera is None or detectorNum is None:
1600 raise RuntimeError('Must supply both a camera and detector number when using exposures '
1601 'without a detector set.')
1602 inputExp.setDetector(camera[detectorNum])
1603
1604 return inputExp
1605
1606 def convertIntToFloat(self, exposure):
1607 """Convert exposure image from uint16 to float.
1608
1609 If the exposure does not need to be converted, the input is
1610 immediately returned. For exposures that are converted to use
1611 floating point pixels, the variance is set to unity and the
1612 mask to zero.
1613
1614 Parameters
1615 ----------
1616 exposure : `lsst.afw.image.Exposure`
1617 The raw exposure to be converted.
1618
1619 Returns
1620 -------
1621 newexposure : `lsst.afw.image.Exposure`
1622 The input ``exposure``, converted to floating point pixels.
1623
1624 Raises
1625 ------
1626 RuntimeError
1627 Raised if the exposure type cannot be converted to float.
1628
1629 """
1630 if isinstance(exposure, afwImage.ExposureF):
1631 # Nothing to be done
1632 self.log.debug("Exposure already of type float.")
1633 return exposure
1634 if not hasattr(exposure, "convertF"):
1635 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1636
1637 newexposure = exposure.convertF()
1638 newexposure.variance[:] = 1
1639 newexposure.mask[:] = 0x0
1640
1641 return newexposure
1642
1643 def maskAmplifier(self, ccdExposure, amp, defects):
1644 """Identify bad amplifiers, saturated and suspect pixels.
1645
1646 Parameters
1647 ----------
1648 ccdExposure : `lsst.afw.image.Exposure`
1649 Input exposure to be masked.
1651 Catalog of parameters defining the amplifier on this
1652 exposure to mask.
1653 defects : `lsst.ip.isr.Defects`
1654 List of defects. Used to determine if the entire
1655 amplifier is bad.
1656
1657 Returns
1658 -------
1659 badAmp : `Bool`
1660 If this is true, the entire amplifier area is covered by
1661 defects and unusable.
1662
1663 """
1664 maskedImage = ccdExposure.getMaskedImage()
1665
1666 badAmp = False
1667
1668 # Check if entire amp region is defined as a defect
1669 # NB: need to use amp.getBBox() for correct comparison with current
1670 # defects definition.
1671 if defects is not None:
1672 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1673
1674 # In the case of a bad amp, we will set mask to "BAD"
1675 # (here use amp.getRawBBox() for correct association with pixels in
1676 # current ccdExposure).
1677 if badAmp:
1678 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1679 afwImage.PARENT)
1680 maskView = dataView.getMask()
1681 maskView |= maskView.getPlaneBitMask("BAD")
1682 del maskView
1683 return badAmp
1684
1685 # Mask remaining defects after assembleCcd() to allow for defects that
1686 # cross amplifier boundaries. Saturation and suspect pixels can be
1687 # masked now, though.
1688 limits = dict()
1689 if self.config.doSaturation and not badAmp:
1690 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1691 if self.config.doSuspect and not badAmp:
1692 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1693 if math.isfinite(self.config.saturation):
1694 limits.update({self.config.saturatedMaskName: self.config.saturation})
1695
1696 for maskName, maskThreshold in limits.items():
1697 if not math.isnan(maskThreshold):
1698 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1699 isrFunctions.makeThresholdMask(
1700 maskedImage=dataView,
1701 threshold=maskThreshold,
1702 growFootprints=0,
1703 maskName=maskName
1704 )
1705
1706 # Determine if we've fully masked this amplifier with SUSPECT and
1707 # SAT pixels.
1708 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1709 afwImage.PARENT)
1710 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1711 self.config.suspectMaskName])
1712 if numpy.all(maskView.getArray() & maskVal > 0):
1713 badAmp = True
1714 maskView |= maskView.getPlaneBitMask("BAD")
1715
1716 return badAmp
1717
1718 def overscanCorrection(self, ccdExposure, amp):
1719 """Apply overscan correction in place.
1720
1721 This method does initial pixel rejection of the overscan
1722 region. The overscan can also be optionally segmented to
1723 allow for discontinuous overscan responses to be fit
1724 separately. The actual overscan subtraction is performed by
1725 the `lsst.ip.isr.overscan.OverscanTask`, which is called here
1726 after the amplifier is preprocessed.
1727
1728 Parameters
1729 ----------
1730 ccdExposure : `lsst.afw.image.Exposure`
1731 Exposure to have overscan correction performed.
1732 amp : `lsst.afw.cameraGeom.Amplifer`
1733 The amplifier to consider while correcting the overscan.
1734
1735 Returns
1736 -------
1737 overscanResults : `lsst.pipe.base.Struct`
1738 Result struct with components:
1739 - ``imageFit`` : scalar or `lsst.afw.image.Image`
1740 Value or fit subtracted from the amplifier image data.
1741 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1742 Value or fit subtracted from the overscan image data.
1743 - ``overscanImage`` : `lsst.afw.image.Image`
1744 Image of the overscan region with the overscan
1745 correction applied. This quantity is used to estimate
1746 the amplifier read noise empirically.
1747 - ``edgeMask`` : `lsst.afw.image.Mask`
1748 Mask of the suspect pixels.
1749 - ``overscanMean`` : `float`
1750 Median overscan fit value.
1751 - ``overscanSigma`` : `float`
1752 Clipped standard deviation of the overscan after
1753 correction.
1754
1755 Raises
1756 ------
1757 RuntimeError
1758 Raised if the ``amp`` does not contain raw pixel information.
1759
1760 See Also
1761 --------
1762 lsst.ip.isr.overscan.OverscanTask
1763
1764 """
1765 if amp.getRawHorizontalOverscanBBox().isEmpty():
1766 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1767 return None
1768
1769 # Perform overscan correction on subregions.
1770 overscanResults = self.overscan.run(ccdExposure, amp)
1771
1772 metadata = ccdExposure.getMetadata()
1773 ampNum = amp.getName()
1774 metadata[f"ISR_OSCAN_LEVEL{ampNum}"] = overscanResults.overscanMean
1775 metadata[f"ISR_OSCAN_SIGMA{ampNum}"] = overscanResults.overscanSigma
1776
1777 return overscanResults
1778
1779 def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None):
1780 """Set the variance plane using the gain and read noise
1781
1782 The read noise is calculated from the ``overscanImage`` if the
1783 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1784 the value from the amplifier data is used.
1785
1786 Parameters
1787 ----------
1788 ampExposure : `lsst.afw.image.Exposure`
1789 Exposure to process.
1790 amp : `lsst.afw.cameraGeom.Amplifier` or `FakeAmp`
1791 Amplifier detector data.
1792 overscanImage : `lsst.afw.image.MaskedImage`, optional.
1793 Image of overscan, required only for empirical read noise.
1794 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
1795 PTC dataset containing the gains and read noise.
1796
1797 Raises
1798 ------
1799 RuntimeError
1800 Raised if either ``usePtcGains`` of ``usePtcReadNoise``
1801 are ``True``, but ptcDataset is not provided.
1802
1803 Raised if ```doEmpiricalReadNoise`` is ``True`` but
1804 ``overscanImage`` is ``None``.
1805
1806 See also
1807 --------
1808 lsst.ip.isr.isrFunctions.updateVariance
1809 """
1810 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1811 if self.config.usePtcGains:
1812 if ptcDataset is None:
1813 raise RuntimeError("No ptcDataset provided to use PTC gains.")
1814 else:
1815 gain = ptcDataset.gain[amp.getName()]
1816 self.log.info("Using gain from Photon Transfer Curve.")
1817 else:
1818 gain = amp.getGain()
1819
1820 if math.isnan(gain):
1821 gain = 1.0
1822 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1823 elif gain <= 0:
1824 patchedGain = 1.0
1825 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.",
1826 amp.getName(), gain, patchedGain)
1827 gain = patchedGain
1828
1829 if self.config.doEmpiricalReadNoise and overscanImage is None:
1830 raise RuntimeError("Overscan is none for EmpiricalReadNoise.")
1831
1832 if self.config.doEmpiricalReadNoise and overscanImage is not None:
1833 stats = afwMath.StatisticsControl()
1834 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
1835 readNoise = afwMath.makeStatistics(overscanImage.getImage(),
1836 afwMath.STDEVCLIP, stats).getValue()
1837 self.log.info("Calculated empirical read noise for amp %s: %f.",
1838 amp.getName(), readNoise)
1839 elif self.config.usePtcReadNoise:
1840 if ptcDataset is None:
1841 raise RuntimeError("No ptcDataset provided to use PTC readnoise.")
1842 else:
1843 readNoise = ptcDataset.noise[amp.getName()]
1844 self.log.info("Using read noise from Photon Transfer Curve.")
1845 else:
1846 readNoise = amp.getReadNoise()
1847
1848 isrFunctions.updateVariance(
1849 maskedImage=ampExposure.getMaskedImage(),
1850 gain=gain,
1851 readNoise=readNoise,
1852 )
1853
1854 def maskNegativeVariance(self, exposure):
1855 """Identify and mask pixels with negative variance values.
1856
1857 Parameters
1858 ----------
1859 exposure : `lsst.afw.image.Exposure`
1860 Exposure to process.
1861
1862 See Also
1863 --------
1864 lsst.ip.isr.isrFunctions.updateVariance
1865 """
1866 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
1867 bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
1868 exposure.mask.array[bad] |= maskPlane
1869
1870 def darkCorrection(self, exposure, darkExposure, invert=False):
1871 """Apply dark correction in place.
1872
1873 Parameters
1874 ----------
1875 exposure : `lsst.afw.image.Exposure`
1876 Exposure to process.
1877 darkExposure : `lsst.afw.image.Exposure`
1878 Dark exposure of the same size as ``exposure``.
1879 invert : `Bool`, optional
1880 If True, re-add the dark to an already corrected image.
1881
1882 Raises
1883 ------
1884 RuntimeError
1885 Raised if either ``exposure`` or ``darkExposure`` do not
1886 have their dark time defined.
1887
1888 See Also
1889 --------
1890 lsst.ip.isr.isrFunctions.darkCorrection
1891 """
1892 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
1893 if math.isnan(expScale):
1894 raise RuntimeError("Exposure darktime is NAN.")
1895 if darkExposure.getInfo().getVisitInfo() is not None \
1896 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
1897 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
1898 else:
1899 # DM-17444: darkExposure.getInfo.getVisitInfo() is None
1900 # so getDarkTime() does not exist.
1901 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
1902 darkScale = 1.0
1903
1904 isrFunctions.darkCorrection(
1905 maskedImage=exposure.getMaskedImage(),
1906 darkMaskedImage=darkExposure.getMaskedImage(),
1907 expScale=expScale,
1908 darkScale=darkScale,
1909 invert=invert,
1910 trimToFit=self.config.doTrimToMatchCalib
1911 )
1912
1913 def doLinearize(self, detector):
1914 """Check if linearization is needed for the detector cameraGeom.
1915
1916 Checks config.doLinearize and the linearity type of the first
1917 amplifier.
1918
1919 Parameters
1920 ----------
1921 detector : `lsst.afw.cameraGeom.Detector`
1922 Detector to get linearity type from.
1923
1924 Returns
1925 -------
1926 doLinearize : `Bool`
1927 If True, linearization should be performed.
1928 """
1929 return self.config.doLinearize and \
1930 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
1931
1932 def flatCorrection(self, exposure, flatExposure, invert=False):
1933 """Apply flat correction in place.
1934
1935 Parameters
1936 ----------
1937 exposure : `lsst.afw.image.Exposure`
1938 Exposure to process.
1939 flatExposure : `lsst.afw.image.Exposure`
1940 Flat exposure of the same size as ``exposure``.
1941 invert : `Bool`, optional
1942 If True, unflatten an already flattened image.
1943
1944 See Also
1945 --------
1946 lsst.ip.isr.isrFunctions.flatCorrection
1947 """
1948 isrFunctions.flatCorrection(
1949 maskedImage=exposure.getMaskedImage(),
1950 flatMaskedImage=flatExposure.getMaskedImage(),
1951 scalingType=self.config.flatScalingType,
1952 userScale=self.config.flatUserScale,
1953 invert=invert,
1954 trimToFit=self.config.doTrimToMatchCalib
1955 )
1956
1957 def saturationDetection(self, exposure, amp):
1958 """Detect and mask saturated pixels in config.saturatedMaskName.
1959
1960 Parameters
1961 ----------
1962 exposure : `lsst.afw.image.Exposure`
1963 Exposure to process. Only the amplifier DataSec is processed.
1965 Amplifier detector data.
1966
1967 See Also
1968 --------
1969 lsst.ip.isr.isrFunctions.makeThresholdMask
1970 """
1971 if not math.isnan(amp.getSaturation()):
1972 maskedImage = exposure.getMaskedImage()
1973 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1974 isrFunctions.makeThresholdMask(
1975 maskedImage=dataView,
1976 threshold=amp.getSaturation(),
1977 growFootprints=0,
1978 maskName=self.config.saturatedMaskName,
1979 )
1980
1981 def saturationInterpolation(self, exposure):
1982 """Interpolate over saturated pixels, in place.
1983
1984 This method should be called after `saturationDetection`, to
1985 ensure that the saturated pixels have been identified in the
1986 SAT mask. It should also be called after `assembleCcd`, since
1987 saturated regions may cross amplifier boundaries.
1988
1989 Parameters
1990 ----------
1991 exposure : `lsst.afw.image.Exposure`
1992 Exposure to process.
1993
1994 See Also
1995 --------
1996 lsst.ip.isr.isrTask.saturationDetection
1997 lsst.ip.isr.isrFunctions.interpolateFromMask
1998 """
1999 isrFunctions.interpolateFromMask(
2000 maskedImage=exposure.getMaskedImage(),
2001 fwhm=self.config.fwhm,
2002 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2003 maskNameList=list(self.config.saturatedMaskName),
2004 )
2005
2006 def suspectDetection(self, exposure, amp):
2007 """Detect and mask suspect pixels in config.suspectMaskName.
2008
2009 Parameters
2010 ----------
2011 exposure : `lsst.afw.image.Exposure`
2012 Exposure to process. Only the amplifier DataSec is processed.
2014 Amplifier detector data.
2015
2016 See Also
2017 --------
2018 lsst.ip.isr.isrFunctions.makeThresholdMask
2019
2020 Notes
2021 -----
2022 Suspect pixels are pixels whose value is greater than
2023 amp.getSuspectLevel(). This is intended to indicate pixels that may be
2024 affected by unknown systematics; for example if non-linearity
2025 corrections above a certain level are unstable then that would be a
2026 useful value for suspectLevel. A value of `nan` indicates that no such
2027 level exists and no pixels are to be masked as suspicious.
2028 """
2029 suspectLevel = amp.getSuspectLevel()
2030 if math.isnan(suspectLevel):
2031 return
2032
2033 maskedImage = exposure.getMaskedImage()
2034 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2035 isrFunctions.makeThresholdMask(
2036 maskedImage=dataView,
2037 threshold=suspectLevel,
2038 growFootprints=0,
2039 maskName=self.config.suspectMaskName,
2040 )
2041
2042 def maskDefect(self, exposure, defectBaseList):
2043 """Mask defects using mask plane "BAD", in place.
2044
2045 Parameters
2046 ----------
2047 exposure : `lsst.afw.image.Exposure`
2048 Exposure to process.
2049 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2051 List of defects to mask.
2052
2053 Notes
2054 -----
2055 Call this after CCD assembly, since defects may cross amplifier
2056 boundaries.
2057 """
2058 maskedImage = exposure.getMaskedImage()
2059 if not isinstance(defectBaseList, Defects):
2060 # Promotes DefectBase to Defect
2061 defectList = Defects(defectBaseList)
2062 else:
2063 defectList = defectBaseList
2064 defectList.maskPixels(maskedImage, maskName="BAD")
2065
2066 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2067 """Mask edge pixels with applicable mask plane.
2068
2069 Parameters
2070 ----------
2071 exposure : `lsst.afw.image.Exposure`
2072 Exposure to process.
2073 numEdgePixels : `int`, optional
2074 Number of edge pixels to mask.
2075 maskPlane : `str`, optional
2076 Mask plane name to use.
2077 level : `str`, optional
2078 Level at which to mask edges.
2079 """
2080 maskedImage = exposure.getMaskedImage()
2081 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2082
2083 if numEdgePixels > 0:
2084 if level == 'DETECTOR':
2085 boxes = [maskedImage.getBBox()]
2086 elif level == 'AMP':
2087 boxes = [amp.getBBox() for amp in exposure.getDetector()]
2088
2089 for box in boxes:
2090 # This makes a bbox numEdgeSuspect pixels smaller than the
2091 # image on each side
2092 subImage = maskedImage[box]
2093 box.grow(-numEdgePixels)
2094 # Mask pixels outside box
2095 SourceDetectionTask.setEdgeBits(
2096 subImage,
2097 box,
2098 maskBitMask)
2099
2100 def maskAndInterpolateDefects(self, exposure, defectBaseList):
2101 """Mask and interpolate defects using mask plane "BAD", in place.
2102
2103 Parameters
2104 ----------
2105 exposure : `lsst.afw.image.Exposure`
2106 Exposure to process.
2107 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2109 List of defects to mask and interpolate.
2110
2111 See Also
2112 --------
2113 lsst.ip.isr.isrTask.maskDefect
2114 """
2115 self.maskDefect(exposure, defectBaseList)
2116 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2117 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2118 isrFunctions.interpolateFromMask(
2119 maskedImage=exposure.getMaskedImage(),
2120 fwhm=self.config.fwhm,
2121 growSaturatedFootprints=0,
2122 maskNameList=["BAD"],
2123 )
2124
2125 def maskNan(self, exposure):
2126 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2127
2128 Parameters
2129 ----------
2130 exposure : `lsst.afw.image.Exposure`
2131 Exposure to process.
2132
2133 Notes
2134 -----
2135 We mask over all non-finite values (NaN, inf), including those
2136 that are masked with other bits (because those may or may not be
2137 interpolated over later, and we want to remove all NaN/infs).
2138 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2139 preserve the historical name.
2140 """
2141 maskedImage = exposure.getMaskedImage()
2142
2143 # Find and mask NaNs
2144 maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2145 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2146 numNans = maskNans(maskedImage, maskVal)
2147 self.metadata["NUMNANS"] = numNans
2148 if numNans > 0:
2149 self.log.warning("There were %d unmasked NaNs.", numNans)
2150
2151 def maskAndInterpolateNan(self, exposure):
2152 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2153 in place.
2154
2155 Parameters
2156 ----------
2157 exposure : `lsst.afw.image.Exposure`
2158 Exposure to process.
2159
2160 See Also
2161 --------
2162 lsst.ip.isr.isrTask.maskNan
2163 """
2164 self.maskNan(exposure)
2165 isrFunctions.interpolateFromMask(
2166 maskedImage=exposure.getMaskedImage(),
2167 fwhm=self.config.fwhm,
2168 growSaturatedFootprints=0,
2169 maskNameList=["UNMASKEDNAN"],
2170 )
2171
2172 def measureBackground(self, exposure, IsrQaConfig=None):
2173 """Measure the image background in subgrids, for quality control.
2174
2175 Parameters
2176 ----------
2177 exposure : `lsst.afw.image.Exposure`
2178 Exposure to process.
2179 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2180 Configuration object containing parameters on which background
2181 statistics and subgrids to use.
2182 """
2183 if IsrQaConfig is not None:
2184 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2185 IsrQaConfig.flatness.nIter)
2186 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2187 statsControl.setAndMask(maskVal)
2188 maskedImage = exposure.getMaskedImage()
2189 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2190 skyLevel = stats.getValue(afwMath.MEDIAN)
2191 skySigma = stats.getValue(afwMath.STDEVCLIP)
2192 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2193 metadata = exposure.getMetadata()
2194 metadata["SKYLEVEL"] = skyLevel
2195 metadata["SKYSIGMA"] = skySigma
2196
2197 # calcluating flatlevel over the subgrids
2198 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2199 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2200 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2201 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2202 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2203 skyLevels = numpy.zeros((nX, nY))
2204
2205 for j in range(nY):
2206 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2207 for i in range(nX):
2208 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2209
2210 xLLC = xc - meshXHalf
2211 yLLC = yc - meshYHalf
2212 xURC = xc + meshXHalf - 1
2213 yURC = yc + meshYHalf - 1
2214
2215 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2216 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2217
2218 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2219
2220 good = numpy.where(numpy.isfinite(skyLevels))
2221 skyMedian = numpy.median(skyLevels[good])
2222 flatness = (skyLevels[good] - skyMedian) / skyMedian
2223 flatness_rms = numpy.std(flatness)
2224 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2225
2226 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2227 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2228 nX, nY, flatness_pp, flatness_rms)
2229
2230 metadata["FLATNESS_PP"] = float(flatness_pp)
2231 metadata["FLATNESS_RMS"] = float(flatness_rms)
2232 metadata["FLATNESS_NGRIDS"] = '%dx%d' % (nX, nY)
2233 metadata["FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX
2234 metadata["FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY
2235
2236 def roughZeroPoint(self, exposure):
2237 """Set an approximate magnitude zero point for the exposure.
2238
2239 Parameters
2240 ----------
2241 exposure : `lsst.afw.image.Exposure`
2242 Exposure to process.
2243 """
2244 filterLabel = exposure.getFilter()
2245 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2246
2247 if physicalFilter in self.config.fluxMag0T1:
2248 fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2249 else:
2250 self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter)
2251 fluxMag0 = self.config.defaultFluxMag0T1
2252
2253 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2254 if not expTime > 0: # handle NaN as well as <= 0
2255 self.log.warning("Non-positive exposure time; skipping rough zero point.")
2256 return
2257
2258 self.log.info("Setting rough magnitude zero point for filter %s: %f",
2259 physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2260 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2261
2262 @contextmanager
2263 def flatContext(self, exp, flat, dark=None):
2264 """Context manager that applies and removes flats and darks,
2265 if the task is configured to apply them.
2266
2267 Parameters
2268 ----------
2270 Exposure to process.
2272 Flat exposure the same size as ``exp``.
2273 dark : `lsst.afw.image.Exposure`, optional
2274 Dark exposure the same size as ``exp``.
2275
2276 Yields
2277 ------
2279 The flat and dark corrected exposure.
2280 """
2281 if self.config.doDark and dark is not None:
2282 self.darkCorrection(exp, dark)
2283 if self.config.doFlat:
2284 self.flatCorrection(exp, flat)
2285 try:
2286 yield exp
2287 finally:
2288 if self.config.doFlat:
2289 self.flatCorrection(exp, flat, invert=True)
2290 if self.config.doDark and dark is not None:
2291 self.darkCorrection(exp, dark, invert=True)
2292
2293 def debugView(self, exposure, stepname):
2294 """Utility function to examine ISR exposure at different stages.
2295
2296 Parameters
2297 ----------
2298 exposure : `lsst.afw.image.Exposure`
2299 Exposure to view.
2300 stepname : `str`
2301 State of processing to view.
2302 """
2303 frame = getDebugFrame(self._display, stepname)
2304 if frame:
2305 display = getDisplay(frame)
2306 display.scale('asinh', 'zscale')
2307 display.mtv(exposure)
2308 prompt = "Press Enter to continue [c]... "
2309 while True:
2310 ans = input(prompt).lower()
2311 if ans in ("", "c",):
2312 break
2313
2314
2315class FakeAmp(object):
2316 """A Detector-like object that supports returning gain and saturation level
2317
2318 This is used when the input exposure does not have a detector.
2319
2320 Parameters
2321 ----------
2322 exposure : `lsst.afw.image.Exposure`
2323 Exposure to generate a fake amplifier for.
2324 config : `lsst.ip.isr.isrTaskConfig`
2325 Configuration to apply to the fake amplifier.
2326 """
2327
2328 def __init__(self, exposure, config):
2329 self._bbox = exposure.getBBox(afwImage.LOCAL)
2331 self._gain = config.gain
2332 self._readNoise = config.readNoise
2333 self._saturation = config.saturation
2334
2335 def getBBox(self):
2336 return self._bbox
2337
2338 def getRawBBox(self):
2339 return self._bbox
2340
2342 return self._RawHorizontalOverscanBBox
2343
2344 def getGain(self):
2345 return self._gain
2346
2347 def getReadNoise(self):
2348 return self._readNoise
2349
2350 def getSaturation(self):
2351 return self._saturation
2352
2354 return float("NaN")
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2341
def __init__(self, exposure, config)
Definition: isrTask.py:2328
def __init__(self, *config=None)
Definition: isrTask.py:283
def flatCorrection(self, exposure, flatExposure, invert=False)
Definition: isrTask.py:1932
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2151
def saturationInterpolation(self, exposure)
Definition: isrTask.py:1981
def maskNan(self, exposure)
Definition: isrTask.py:2125
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1643
def debugView(self, exposure, stepname)
Definition: isrTask.py:2293
def ensureExposure(self, inputExp, camera=None, detectorNum=None)
Definition: isrTask.py:1554
def maskNegativeVariance(self, exposure)
Definition: isrTask.py:1854
def saturationDetection(self, exposure, amp)
Definition: isrTask.py:1957
def maskDefect(self, exposure, defectBaseList)
Definition: isrTask.py:2042
def __init__(self, **kwargs)
Definition: isrTask.py:942
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:955
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
Definition: isrTask.py:2066
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1718
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, deferredCharge=None)
Definition: isrTask.py:1060
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2172
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2236
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2100
def doLinearize(self, detector)
Definition: isrTask.py:1913
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2263
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1606
def suspectDetection(self, exposure, amp)
Definition: isrTask.py:2006
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
Definition: isrTask.py:1779
def darkCorrection(self, exposure, darkExposure, invert=False)
Definition: isrTask.py:1870
def checkFilter(exposure, filterList, log)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
Definition: isrTask.py:62
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35