lsst.ip.isr g673fa65101+0a9654e073
isrTask.py
Go to the documentation of this file.
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22import math
23import numpy
24
25import lsst.geom
26import lsst.afw.image as afwImage
27import lsst.afw.math as afwMath
28import lsst.pex.config as pexConfig
29import lsst.pipe.base as pipeBase
30import lsst.pipe.base.connectionTypes as cT
31
32from contextlib import contextmanager
33from lsstDebug import getDebugFrame
34
35from lsst.afw.cameraGeom import NullLinearityType, ReadoutCorner
36from lsst.afw.display import getDisplay
37from lsst.meas.algorithms.detection import SourceDetectionTask
38from lsst.utils.timer import timeMethod
39
40from . import isrFunctions
41from . import isrQa
42from . import linearize
43from .defects import Defects
44
45from .assembleCcdTask import AssembleCcdTask
46from .crosstalk import CrosstalkTask, CrosstalkCalib
47from .fringe import FringeTask
48from .isr import maskNans
49from .masking import MaskingTask
50from .overscan import OverscanCorrectionTask
51from .straylight import StrayLightTask
52from .vignette import VignetteTask
53from .ampOffset import AmpOffsetTask
54from .deferredCharge import DeferredChargeTask
55from .isrStatistics import IsrStatisticsTask
56from lsst.daf.butler import DimensionGraph
57
58
59__all__ = ["IsrTask", "IsrTaskConfig"]
60
61
62def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections):
63 """Lookup function to identify crosstalkSource entries.
64
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
67 populated.
68
69 Parameters
70 ----------
71 datasetType : `str`
72 Dataset to lookup.
73 registry : `lsst.daf.butler.Registry`
74 Butler registry to query.
75 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
76 Data id to transform to identify crosstalkSources. The
77 ``detector`` entry will be stripped.
78 collections : `lsst.daf.butler.CollectionSearch`
79 Collections to search through.
80
81 Returns
82 -------
83 results : `list` [`lsst.daf.butler.DatasetRef`]
84 List of datasets that match the query that will be used as
85 crosstalkSources.
86 """
87 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"]))
88 results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId,
89 findFirst=True))
90 # In some contexts, calling `.expanded()` to expand all data IDs in the
91 # query results can be a lot faster because it vectorizes lookups. But in
92 # this case, expandDataId shouldn't need to hit the database at all in the
93 # steady state, because only the detector record is unknown and those are
94 # cached in the registry.
95 return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records)) for ref in results]
96
97
98class IsrTaskConnections(pipeBase.PipelineTaskConnections,
99 dimensions={"instrument", "exposure", "detector"},
100 defaultTemplates={}):
101 ccdExposure = cT.Input(
102 name="raw",
103 doc="Input exposure to process.",
104 storageClass="Exposure",
105 dimensions=["instrument", "exposure", "detector"],
106 )
107 camera = cT.PrerequisiteInput(
108 name="camera",
109 storageClass="Camera",
110 doc="Input camera to construct complete exposures.",
111 dimensions=["instrument"],
112 isCalibration=True,
113 )
114
115 crosstalk = cT.PrerequisiteInput(
116 name="crosstalk",
117 doc="Input crosstalk object",
118 storageClass="CrosstalkCalib",
119 dimensions=["instrument", "detector"],
120 isCalibration=True,
121 minimum=0, # can fall back to cameraGeom
122 )
123 crosstalkSources = cT.PrerequisiteInput(
124 name="isrOverscanCorrected",
125 doc="Overscan corrected input images.",
126 storageClass="Exposure",
127 dimensions=["instrument", "exposure", "detector"],
128 deferLoad=True,
129 multiple=True,
130 lookupFunction=crosstalkSourceLookup,
131 minimum=0, # not needed for all instruments, no config to control this
132 )
133 bias = cT.PrerequisiteInput(
134 name="bias",
135 doc="Input bias calibration.",
136 storageClass="ExposureF",
137 dimensions=["instrument", "detector"],
138 isCalibration=True,
139 )
140 dark = cT.PrerequisiteInput(
141 name='dark',
142 doc="Input dark calibration.",
143 storageClass="ExposureF",
144 dimensions=["instrument", "detector"],
145 isCalibration=True,
146 )
147 flat = cT.PrerequisiteInput(
148 name="flat",
149 doc="Input flat calibration.",
150 storageClass="ExposureF",
151 dimensions=["instrument", "physical_filter", "detector"],
152 isCalibration=True,
153 )
154 ptc = cT.PrerequisiteInput(
155 name="ptc",
156 doc="Input Photon Transfer Curve dataset",
157 storageClass="PhotonTransferCurveDataset",
158 dimensions=["instrument", "detector"],
159 isCalibration=True,
160 )
161 fringes = cT.PrerequisiteInput(
162 name="fringe",
163 doc="Input fringe calibration.",
164 storageClass="ExposureF",
165 dimensions=["instrument", "physical_filter", "detector"],
166 isCalibration=True,
167 minimum=0, # only needed for some bands, even when enabled
168 )
169 strayLightData = cT.PrerequisiteInput(
170 name='yBackground',
171 doc="Input stray light calibration.",
172 storageClass="StrayLightData",
173 dimensions=["instrument", "physical_filter", "detector"],
174 deferLoad=True,
175 isCalibration=True,
176 minimum=0, # only needed for some bands, even when enabled
177 )
178 bfKernel = cT.PrerequisiteInput(
179 name='bfKernel',
180 doc="Input brighter-fatter kernel.",
181 storageClass="NumpyArray",
182 dimensions=["instrument"],
183 isCalibration=True,
184 minimum=0, # can use either bfKernel or newBFKernel
185 )
186 newBFKernel = cT.PrerequisiteInput(
187 name='brighterFatterKernel',
188 doc="Newer complete kernel + gain solutions.",
189 storageClass="BrighterFatterKernel",
190 dimensions=["instrument", "detector"],
191 isCalibration=True,
192 minimum=0, # can use either bfKernel or newBFKernel
193 )
194 defects = cT.PrerequisiteInput(
195 name='defects',
196 doc="Input defect tables.",
197 storageClass="Defects",
198 dimensions=["instrument", "detector"],
199 isCalibration=True,
200 )
201 linearizer = cT.PrerequisiteInput(
202 name='linearizer',
203 storageClass="Linearizer",
204 doc="Linearity correction calibration.",
205 dimensions=["instrument", "detector"],
206 isCalibration=True,
207 minimum=0, # can fall back to cameraGeom
208 )
209 opticsTransmission = cT.PrerequisiteInput(
210 name="transmission_optics",
211 storageClass="TransmissionCurve",
212 doc="Transmission curve due to the optics.",
213 dimensions=["instrument"],
214 isCalibration=True,
215 )
216 filterTransmission = cT.PrerequisiteInput(
217 name="transmission_filter",
218 storageClass="TransmissionCurve",
219 doc="Transmission curve due to the filter.",
220 dimensions=["instrument", "physical_filter"],
221 isCalibration=True,
222 )
223 sensorTransmission = cT.PrerequisiteInput(
224 name="transmission_sensor",
225 storageClass="TransmissionCurve",
226 doc="Transmission curve due to the sensor.",
227 dimensions=["instrument", "detector"],
228 isCalibration=True,
229 )
230 atmosphereTransmission = cT.PrerequisiteInput(
231 name="transmission_atmosphere",
232 storageClass="TransmissionCurve",
233 doc="Transmission curve due to the atmosphere.",
234 dimensions=["instrument"],
235 isCalibration=True,
236 )
237 illumMaskedImage = cT.PrerequisiteInput(
238 name="illum",
239 doc="Input illumination correction.",
240 storageClass="MaskedImageF",
241 dimensions=["instrument", "physical_filter", "detector"],
242 isCalibration=True,
243 )
244 deferredChargeCalib = cT.PrerequisiteInput(
245 name="deferredCharge",
246 doc="Deferred charge/CTI correction dataset.",
247 storageClass="IsrCalib",
248 dimensions=["instrument", "detector"],
249 isCalibration=True,
250 )
251
252 outputExposure = cT.Output(
253 name='postISRCCD',
254 doc="Output ISR processed exposure.",
255 storageClass="Exposure",
256 dimensions=["instrument", "exposure", "detector"],
257 )
258 preInterpExposure = cT.Output(
259 name='preInterpISRCCD',
260 doc="Output ISR processed exposure, with pixels left uninterpolated.",
261 storageClass="ExposureF",
262 dimensions=["instrument", "exposure", "detector"],
263 )
264 outputOssThumbnail = cT.Output(
265 name="OssThumb",
266 doc="Output Overscan-subtracted thumbnail image.",
267 storageClass="Thumbnail",
268 dimensions=["instrument", "exposure", "detector"],
269 )
270 outputFlattenedThumbnail = cT.Output(
271 name="FlattenedThumb",
272 doc="Output flat-corrected thumbnail image.",
273 storageClass="Thumbnail",
274 dimensions=["instrument", "exposure", "detector"],
275 )
276 outputStatistics = cT.Output(
277 name="isrStatistics",
278 doc="Output of additional statistics table.",
279 storageClass="StructuredDataDict",
280 dimensions=["instrument", "exposure", "detector"],
281 )
282
283 def __init__(self, *, config=None):
284 super().__init__(config=config)
285
286 if config.doBias is not True:
287 self.prerequisiteInputs.remove("bias")
288 if config.doLinearize is not True:
289 self.prerequisiteInputs.remove("linearizer")
290 if config.doCrosstalk is not True:
291 self.prerequisiteInputs.remove("crosstalkSources")
292 self.prerequisiteInputs.remove("crosstalk")
293 if config.doBrighterFatter is not True:
294 self.prerequisiteInputs.remove("bfKernel")
295 self.prerequisiteInputs.remove("newBFKernel")
296 if config.doDefect is not True:
297 self.prerequisiteInputs.remove("defects")
298 if config.doDark is not True:
299 self.prerequisiteInputs.remove("dark")
300 if config.doFlat is not True:
301 self.prerequisiteInputs.remove("flat")
302 if config.doFringe is not True:
303 self.prerequisiteInputs.remove("fringes")
304 if config.doStrayLight is not True:
305 self.prerequisiteInputs.remove("strayLightData")
306 if config.usePtcGains is not True and config.usePtcReadNoise is not True:
307 self.prerequisiteInputs.remove("ptc")
308 if config.doAttachTransmissionCurve is not True:
309 self.prerequisiteInputs.remove("opticsTransmission")
310 self.prerequisiteInputs.remove("filterTransmission")
311 self.prerequisiteInputs.remove("sensorTransmission")
312 self.prerequisiteInputs.remove("atmosphereTransmission")
313 else:
314 if config.doUseOpticsTransmission is not True:
315 self.prerequisiteInputs.remove("opticsTransmission")
316 if config.doUseFilterTransmission is not True:
317 self.prerequisiteInputs.remove("filterTransmission")
318 if config.doUseSensorTransmission is not True:
319 self.prerequisiteInputs.remove("sensorTransmission")
320 if config.doUseAtmosphereTransmission is not True:
321 self.prerequisiteInputs.remove("atmosphereTransmission")
322 if config.doIlluminationCorrection is not True:
323 self.prerequisiteInputs.remove("illumMaskedImage")
324 if config.doDeferredCharge is not True:
325 self.prerequisiteInputs.remove("deferredChargeCalib")
326
327 if config.doWrite is not True:
328 self.outputs.remove("outputExposure")
329 self.outputs.remove("preInterpExposure")
330 self.outputs.remove("outputFlattenedThumbnail")
331 self.outputs.remove("outputOssThumbnail")
332 self.outputs.remove("outputStatistics")
333
334 if config.doSaveInterpPixels is not True:
335 self.outputs.remove("preInterpExposure")
336 if config.qa.doThumbnailOss is not True:
337 self.outputs.remove("outputOssThumbnail")
338 if config.qa.doThumbnailFlattened is not True:
339 self.outputs.remove("outputFlattenedThumbnail")
340 if config.doCalculateStatistics is not True:
341 self.outputs.remove("outputStatistics")
342
343
344class IsrTaskConfig(pipeBase.PipelineTaskConfig,
345 pipelineConnections=IsrTaskConnections):
346 """Configuration parameters for IsrTask.
347
348 Items are grouped in the order in which they are executed by the task.
349 """
350 datasetType = pexConfig.Field(
351 dtype=str,
352 doc="Dataset type for input data; users will typically leave this alone, "
353 "but camera-specific ISR tasks will override it",
354 default="raw",
355 )
356
357 fallbackFilterName = pexConfig.Field(
358 dtype=str,
359 doc="Fallback default filter name for calibrations.",
360 optional=True
361 )
362 useFallbackDate = pexConfig.Field(
363 dtype=bool,
364 doc="Pass observation date when using fallback filter.",
365 default=False,
366 )
367 expectWcs = pexConfig.Field(
368 dtype=bool,
369 default=True,
370 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
371 )
372 fwhm = pexConfig.Field(
373 dtype=float,
374 doc="FWHM of PSF in arcseconds.",
375 default=1.0,
376 )
377 qa = pexConfig.ConfigField(
378 dtype=isrQa.IsrQaConfig,
379 doc="QA related configuration options.",
380 )
381
382 # Image conversion configuration
383 doConvertIntToFloat = pexConfig.Field(
384 dtype=bool,
385 doc="Convert integer raw images to floating point values?",
386 default=True,
387 )
388
389 # Saturated pixel handling.
390 doSaturation = pexConfig.Field(
391 dtype=bool,
392 doc="Mask saturated pixels? NB: this is totally independent of the"
393 " interpolation option - this is ONLY setting the bits in the mask."
394 " To have them interpolated make sure doSaturationInterpolation=True",
395 default=True,
396 )
397 saturatedMaskName = pexConfig.Field(
398 dtype=str,
399 doc="Name of mask plane to use in saturation detection and interpolation",
400 default="SAT",
401 )
402 saturation = pexConfig.Field(
403 dtype=float,
404 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
405 default=float("NaN"),
406 )
407 growSaturationFootprintSize = pexConfig.Field(
408 dtype=int,
409 doc="Number of pixels by which to grow the saturation footprints",
410 default=1,
411 )
412
413 # Suspect pixel handling.
414 doSuspect = pexConfig.Field(
415 dtype=bool,
416 doc="Mask suspect pixels?",
417 default=False,
418 )
419 suspectMaskName = pexConfig.Field(
420 dtype=str,
421 doc="Name of mask plane to use for suspect pixels",
422 default="SUSPECT",
423 )
424 numEdgeSuspect = pexConfig.Field(
425 dtype=int,
426 doc="Number of edge pixels to be flagged as untrustworthy.",
427 default=0,
428 )
429 edgeMaskLevel = pexConfig.ChoiceField(
430 dtype=str,
431 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
432 default="DETECTOR",
433 allowed={
434 'DETECTOR': 'Mask only the edges of the full detector.',
435 'AMP': 'Mask edges of each amplifier.',
436 },
437 )
438
439 # Initial masking options.
440 doSetBadRegions = pexConfig.Field(
441 dtype=bool,
442 doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
443 default=True,
444 )
445 badStatistic = pexConfig.ChoiceField(
446 dtype=str,
447 doc="How to estimate the average value for BAD regions.",
448 default='MEANCLIP',
449 allowed={
450 "MEANCLIP": "Correct using the (clipped) mean of good data",
451 "MEDIAN": "Correct using the median of the good data",
452 },
453 )
454
455 # Overscan subtraction configuration.
456 doOverscan = pexConfig.Field(
457 dtype=bool,
458 doc="Do overscan subtraction?",
459 default=True,
460 )
461 overscan = pexConfig.ConfigurableField(
462 target=OverscanCorrectionTask,
463 doc="Overscan subtraction task for image segments.",
464 )
465 overscanFitType = pexConfig.ChoiceField(
466 dtype=str,
467 doc="The method for fitting the overscan bias level.",
468 default='MEDIAN',
469 allowed={
470 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
471 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
472 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
473 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
474 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
475 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
476 "MEAN": "Correct using the mean of the overscan region",
477 "MEANCLIP": "Correct using a clipped mean of the overscan region",
478 "MEDIAN": "Correct using the median of the overscan region",
479 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
480 },
481 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
482 " This option will no longer be used, and will be removed after v20.")
483 )
484 overscanOrder = pexConfig.Field(
485 dtype=int,
486 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, "
487 "or number of spline knots if overscan fit type is a spline."),
488 default=1,
489 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
490 " This option will no longer be used, and will be removed after v20.")
491 )
492 overscanNumSigmaClip = pexConfig.Field(
493 dtype=float,
494 doc="Rejection threshold (sigma) for collapsing overscan before fit",
495 default=3.0,
496 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
497 " This option will no longer be used, and will be removed after v20.")
498 )
499 overscanIsInt = pexConfig.Field(
500 dtype=bool,
501 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
502 " and overscan.FitType=MEDIAN_PER_ROW.",
503 default=True,
504 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
505 " This option will no longer be used, and will be removed after v20.")
506 )
507 # These options do not get deprecated, as they define how we slice up the
508 # image data.
509 overscanNumLeadingColumnsToSkip = pexConfig.Field(
510 dtype=int,
511 doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
512 default=0,
513 )
514 overscanNumTrailingColumnsToSkip = pexConfig.Field(
515 dtype=int,
516 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
517 default=0,
518 )
519 overscanMaxDev = pexConfig.Field(
520 dtype=float,
521 doc="Maximum deviation from the median for overscan",
522 default=1000.0, check=lambda x: x > 0
523 )
524 overscanBiasJump = pexConfig.Field(
525 dtype=bool,
526 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
527 default=False,
528 )
529 overscanBiasJumpKeyword = pexConfig.Field(
530 dtype=str,
531 doc="Header keyword containing information about devices.",
532 default="NO_SUCH_KEY",
533 )
534 overscanBiasJumpDevices = pexConfig.ListField(
535 dtype=str,
536 doc="List of devices that need piecewise overscan correction.",
537 default=(),
538 )
539 overscanBiasJumpLocation = pexConfig.Field(
540 dtype=int,
541 doc="Location of bias jump along y-axis.",
542 default=0,
543 )
544
545 # Amplifier to CCD assembly configuration
546 doAssembleCcd = pexConfig.Field(
547 dtype=bool,
548 default=True,
549 doc="Assemble amp-level exposures into a ccd-level exposure?"
550 )
551 assembleCcd = pexConfig.ConfigurableField(
552 target=AssembleCcdTask,
553 doc="CCD assembly task",
554 )
555
556 # General calibration configuration.
557 doAssembleIsrExposures = pexConfig.Field(
558 dtype=bool,
559 default=False,
560 doc="Assemble amp-level calibration exposures into ccd-level exposure?"
561 )
562 doTrimToMatchCalib = pexConfig.Field(
563 dtype=bool,
564 default=False,
565 doc="Trim raw data to match calibration bounding boxes?"
566 )
567
568 # Bias subtraction.
569 doBias = pexConfig.Field(
570 dtype=bool,
571 doc="Apply bias frame correction?",
572 default=True,
573 )
574 biasDataProductName = pexConfig.Field(
575 dtype=str,
576 doc="Name of the bias data product",
577 default="bias",
578 )
579 doBiasBeforeOverscan = pexConfig.Field(
580 dtype=bool,
581 doc="Reverse order of overscan and bias correction.",
582 default=False
583 )
584
585 # Deferred charge correction.
586 doDeferredCharge = pexConfig.Field(
587 dtype=bool,
588 doc="Apply deferred charge correction?",
589 default=False,
590 )
591 deferredChargeCorrection = pexConfig.ConfigurableField(
592 target=DeferredChargeTask,
593 doc="Deferred charge correction task.",
594 )
595
596 # Variance construction
597 doVariance = pexConfig.Field(
598 dtype=bool,
599 doc="Calculate variance?",
600 default=True
601 )
602 gain = pexConfig.Field(
603 dtype=float,
604 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
605 default=float("NaN"),
606 )
607 readNoise = pexConfig.Field(
608 dtype=float,
609 doc="The read noise to use if no Detector is present in the Exposure",
610 default=0.0,
611 )
612 doEmpiricalReadNoise = pexConfig.Field(
613 dtype=bool,
614 default=False,
615 doc="Calculate empirical read noise instead of value from AmpInfo data?"
616 )
617 usePtcReadNoise = pexConfig.Field(
618 dtype=bool,
619 default=False,
620 doc="Use readnoise values from the Photon Transfer Curve?"
621 )
622 maskNegativeVariance = pexConfig.Field(
623 dtype=bool,
624 default=True,
625 doc="Mask pixels that claim a negative variance? This likely indicates a failure "
626 "in the measurement of the overscan at an edge due to the data falling off faster "
627 "than the overscan model can account for it."
628 )
629 negativeVarianceMaskName = pexConfig.Field(
630 dtype=str,
631 default="BAD",
632 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
633 )
634 # Linearization.
635 doLinearize = pexConfig.Field(
636 dtype=bool,
637 doc="Correct for nonlinearity of the detector's response?",
638 default=True,
639 )
640
641 # Crosstalk.
642 doCrosstalk = pexConfig.Field(
643 dtype=bool,
644 doc="Apply intra-CCD crosstalk correction?",
645 default=False,
646 )
647 doCrosstalkBeforeAssemble = pexConfig.Field(
648 dtype=bool,
649 doc="Apply crosstalk correction before CCD assembly, and before trimming?",
650 default=False,
651 )
652 crosstalk = pexConfig.ConfigurableField(
653 target=CrosstalkTask,
654 doc="Intra-CCD crosstalk correction",
655 )
656
657 # Masking options.
658 doDefect = pexConfig.Field(
659 dtype=bool,
660 doc="Apply correction for CCD defects, e.g. hot pixels?",
661 default=True,
662 )
663 doNanMasking = pexConfig.Field(
664 dtype=bool,
665 doc="Mask non-finite (NAN, inf) pixels?",
666 default=True,
667 )
668 doWidenSaturationTrails = pexConfig.Field(
669 dtype=bool,
670 doc="Widen bleed trails based on their width?",
671 default=True
672 )
673
674 # Brighter-Fatter correction.
675 doBrighterFatter = pexConfig.Field(
676 dtype=bool,
677 default=False,
678 doc="Apply the brighter-fatter correction?"
679 )
680 brighterFatterLevel = pexConfig.ChoiceField(
681 dtype=str,
682 default="DETECTOR",
683 doc="The level at which to correct for brighter-fatter.",
684 allowed={
685 "AMP": "Every amplifier treated separately.",
686 "DETECTOR": "One kernel per detector",
687 }
688 )
689 brighterFatterMaxIter = pexConfig.Field(
690 dtype=int,
691 default=10,
692 doc="Maximum number of iterations for the brighter-fatter correction"
693 )
694 brighterFatterThreshold = pexConfig.Field(
695 dtype=float,
696 default=1000,
697 doc="Threshold used to stop iterating the brighter-fatter correction. It is the "
698 "absolute value of the difference between the current corrected image and the one "
699 "from the previous iteration summed over all the pixels."
700 )
701 brighterFatterApplyGain = pexConfig.Field(
702 dtype=bool,
703 default=True,
704 doc="Should the gain be applied when applying the brighter-fatter correction?"
705 )
706 brighterFatterMaskListToInterpolate = pexConfig.ListField(
707 dtype=str,
708 doc="List of mask planes that should be interpolated over when applying the brighter-fatter "
709 "correction.",
710 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"],
711 )
712 brighterFatterMaskGrowSize = pexConfig.Field(
713 dtype=int,
714 default=0,
715 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
716 "when brighter-fatter correction is applied."
717 )
718
719 # Dark subtraction.
720 doDark = pexConfig.Field(
721 dtype=bool,
722 doc="Apply dark frame correction?",
723 default=True,
724 )
725 darkDataProductName = pexConfig.Field(
726 dtype=str,
727 doc="Name of the dark data product",
728 default="dark",
729 )
730
731 # Camera-specific stray light removal.
732 doStrayLight = pexConfig.Field(
733 dtype=bool,
734 doc="Subtract stray light in the y-band (due to encoder LEDs)?",
735 default=False,
736 )
737 strayLight = pexConfig.ConfigurableField(
738 target=StrayLightTask,
739 doc="y-band stray light correction"
740 )
741
742 # Flat correction.
743 doFlat = pexConfig.Field(
744 dtype=bool,
745 doc="Apply flat field correction?",
746 default=True,
747 )
748 flatDataProductName = pexConfig.Field(
749 dtype=str,
750 doc="Name of the flat data product",
751 default="flat",
752 )
753 flatScalingType = pexConfig.ChoiceField(
754 dtype=str,
755 doc="The method for scaling the flat on the fly.",
756 default='USER',
757 allowed={
758 "USER": "Scale by flatUserScale",
759 "MEAN": "Scale by the inverse of the mean",
760 "MEDIAN": "Scale by the inverse of the median",
761 },
762 )
763 flatUserScale = pexConfig.Field(
764 dtype=float,
765 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
766 default=1.0,
767 )
768 doTweakFlat = pexConfig.Field(
769 dtype=bool,
770 doc="Tweak flats to match observed amplifier ratios?",
771 default=False
772 )
773
774 # Amplifier normalization based on gains instead of using flats
775 # configuration.
776 doApplyGains = pexConfig.Field(
777 dtype=bool,
778 doc="Correct the amplifiers for their gains instead of applying flat correction",
779 default=False,
780 )
781 usePtcGains = pexConfig.Field(
782 dtype=bool,
783 doc="Use the gain values from the Photon Transfer Curve?",
784 default=False,
785 )
786 normalizeGains = pexConfig.Field(
787 dtype=bool,
788 doc="Normalize all the amplifiers in each CCD to have the same median value.",
789 default=False,
790 )
791
792 # Fringe correction.
793 doFringe = pexConfig.Field(
794 dtype=bool,
795 doc="Apply fringe correction?",
796 default=True,
797 )
798 fringe = pexConfig.ConfigurableField(
799 target=FringeTask,
800 doc="Fringe subtraction task",
801 )
802 fringeAfterFlat = pexConfig.Field(
803 dtype=bool,
804 doc="Do fringe subtraction after flat-fielding?",
805 default=True,
806 )
807
808 # Amp offset correction.
809 doAmpOffset = pexConfig.Field(
810 doc="Calculate and apply amp offset corrections?",
811 dtype=bool,
812 default=False,
813 )
814 ampOffset = pexConfig.ConfigurableField(
815 doc="Amp offset correction task.",
816 target=AmpOffsetTask,
817 )
818
819 # Initial CCD-level background statistics options.
820 doMeasureBackground = pexConfig.Field(
821 dtype=bool,
822 doc="Measure the background level on the reduced image?",
823 default=False,
824 )
825
826 # Camera-specific masking configuration.
827 doCameraSpecificMasking = pexConfig.Field(
828 dtype=bool,
829 doc="Mask camera-specific bad regions?",
830 default=False,
831 )
832 masking = pexConfig.ConfigurableField(
833 target=MaskingTask,
834 doc="Masking task."
835 )
836
837 # Interpolation options.
838 doInterpolate = pexConfig.Field(
839 dtype=bool,
840 doc="Interpolate masked pixels?",
841 default=True,
842 )
843 doSaturationInterpolation = pexConfig.Field(
844 dtype=bool,
845 doc="Perform interpolation over pixels masked as saturated?"
846 " NB: This is independent of doSaturation; if that is False this plane"
847 " will likely be blank, resulting in a no-op here.",
848 default=True,
849 )
850 doNanInterpolation = pexConfig.Field(
851 dtype=bool,
852 doc="Perform interpolation over pixels masked as NaN?"
853 " NB: This is independent of doNanMasking; if that is False this plane"
854 " will likely be blank, resulting in a no-op here.",
855 default=True,
856 )
857 doNanInterpAfterFlat = pexConfig.Field(
858 dtype=bool,
859 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
860 "also have to interpolate them before flat-fielding."),
861 default=False,
862 )
863 maskListToInterpolate = pexConfig.ListField(
864 dtype=str,
865 doc="List of mask planes that should be interpolated.",
866 default=['SAT', 'BAD'],
867 )
868 doSaveInterpPixels = pexConfig.Field(
869 dtype=bool,
870 doc="Save a copy of the pre-interpolated pixel values?",
871 default=False,
872 )
873
874 # Default photometric calibration options.
875 fluxMag0T1 = pexConfig.DictField(
876 keytype=str,
877 itemtype=float,
878 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
879 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
880 ))
881 )
882 defaultFluxMag0T1 = pexConfig.Field(
883 dtype=float,
884 doc="Default value for fluxMag0T1 (for an unrecognized filter).",
885 default=pow(10.0, 0.4*28.0)
886 )
887
888 # Vignette correction configuration.
889 doVignette = pexConfig.Field(
890 dtype=bool,
891 doc=("Compute and attach the validPolygon defining the unvignetted region to the exposure "
892 "according to vignetting parameters?"),
893 default=False,
894 )
895 doMaskVignettePolygon = pexConfig.Field(
896 dtype=bool,
897 doc=("Add a mask bit for pixels within the vignetted region. Ignored if doVignette "
898 "is False"),
899 default=True,
900 )
901 vignetteValue = pexConfig.Field(
902 dtype=float,
903 doc="Value to replace image array pixels with in the vignetted region? Ignored if None.",
904 optional=True,
905 default=None,
906 )
907 vignette = pexConfig.ConfigurableField(
908 target=VignetteTask,
909 doc="Vignetting task.",
910 )
911
912 # Transmission curve configuration.
913 doAttachTransmissionCurve = pexConfig.Field(
914 dtype=bool,
915 default=False,
916 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
917 )
918 doUseOpticsTransmission = pexConfig.Field(
919 dtype=bool,
920 default=True,
921 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
922 )
923 doUseFilterTransmission = pexConfig.Field(
924 dtype=bool,
925 default=True,
926 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
927 )
928 doUseSensorTransmission = pexConfig.Field(
929 dtype=bool,
930 default=True,
931 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
932 )
933 doUseAtmosphereTransmission = pexConfig.Field(
934 dtype=bool,
935 default=True,
936 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
937 )
938
939 # Illumination correction.
940 doIlluminationCorrection = pexConfig.Field(
941 dtype=bool,
942 default=False,
943 doc="Perform illumination correction?"
944 )
945 illuminationCorrectionDataProductName = pexConfig.Field(
946 dtype=str,
947 doc="Name of the illumination correction data product.",
948 default="illumcor",
949 )
950 illumScale = pexConfig.Field(
951 dtype=float,
952 doc="Scale factor for the illumination correction.",
953 default=1.0,
954 )
955 illumFilters = pexConfig.ListField(
956 dtype=str,
957 default=[],
958 doc="Only perform illumination correction for these filters."
959 )
960
961 # Calculate additional statistics?
962 doCalculateStatistics = pexConfig.Field(
963 dtype=bool,
964 doc="Should additional ISR statistics be calculated?",
965 default=False,
966 )
967 isrStats = pexConfig.ConfigurableField(
968 target=IsrStatisticsTask,
969 doc="Task to calculate additional statistics.",
970 )
971
972 # Write the outputs to disk. If ISR is run as a subtask, this may not
973 # be needed.
974 doWrite = pexConfig.Field(
975 dtype=bool,
976 doc="Persist postISRCCD?",
977 default=True,
978 )
979
980 def validate(self):
981 super().validate()
982 if self.doFlat and self.doApplyGains:
983 raise ValueError("You may not specify both doFlat and doApplyGains")
985 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
990 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
991 self.maskListToInterpolate.append("UNMASKEDNAN")
992
993
994class IsrTask(pipeBase.PipelineTask):
995 """Apply common instrument signature correction algorithms to a raw frame.
996
997 The process for correcting imaging data is very similar from
998 camera to camera. This task provides a vanilla implementation of
999 doing these corrections, including the ability to turn certain
1000 corrections off if they are not needed. The inputs to the primary
1001 method, `run()`, are a raw exposure to be corrected and the
1002 calibration data products. The raw input is a single chip sized
1003 mosaic of all amps including overscans and other non-science
1004 pixels.
1005
1006 The __init__ method sets up the subtasks for ISR processing, using
1007 the defaults from `lsst.ip.isr`.
1008
1009 Parameters
1010 ----------
1011 args : `list`
1012 Positional arguments passed to the Task constructor.
1013 None used at this time.
1014 kwargs : `dict`, optional
1015 Keyword arguments passed on to the Task constructor.
1016 None used at this time.
1017 """
1018 ConfigClass = IsrTaskConfig
1019 _DefaultName = "isr"
1020
1021 def __init__(self, **kwargs):
1022 super().__init__(**kwargs)
1023 self.makeSubtask("assembleCcd")
1024 self.makeSubtask("crosstalk")
1025 self.makeSubtask("strayLight")
1026 self.makeSubtask("fringe")
1027 self.makeSubtask("masking")
1028 self.makeSubtask("overscan")
1029 self.makeSubtask("vignette")
1030 self.makeSubtask("ampOffset")
1031 self.makeSubtask("deferredChargeCorrection")
1032 self.makeSubtask("isrStats")
1033
1034 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1035 inputs = butlerQC.get(inputRefs)
1036
1037 try:
1038 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
1039 except Exception as e:
1040 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
1041 (inputRefs, e))
1042
1043 detector = inputs['ccdExposure'].getDetector()
1044
1045 if self.config.doCrosstalk is True:
1046 # Crosstalk sources need to be defined by the pipeline
1047 # yaml if they exist.
1048 if 'crosstalk' in inputs and inputs['crosstalk'] is not None:
1049 if not isinstance(inputs['crosstalk'], CrosstalkCalib):
1050 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk'])
1051 else:
1052 coeffVector = (self.config.crosstalk.crosstalkValues
1053 if self.config.crosstalk.useConfigCoefficients else None)
1054 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
1055 inputs['crosstalk'] = crosstalkCalib
1056 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0:
1057 if 'crosstalkSources' not in inputs:
1058 self.log.warning("No crosstalkSources found for chip with interChip terms!")
1059
1060 if self.doLinearize(detector) is True:
1061 if 'linearizer' in inputs:
1062 if isinstance(inputs['linearizer'], dict):
1063 linearizer = linearize.Linearizer(detector=detector, log=self.log)
1064 linearizer.fromYaml(inputs['linearizer'])
1065 self.log.warning("Dictionary linearizers will be deprecated in DM-28741.")
1066 elif isinstance(inputs['linearizer'], numpy.ndarray):
1067 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None),
1068 detector=detector,
1069 log=self.log)
1070 self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.")
1071 else:
1072 linearizer = inputs['linearizer']
1073 linearizer.log = self.log
1074 inputs['linearizer'] = linearizer
1075 else:
1076 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log)
1077 self.log.warning("Constructing linearizer from cameraGeom information.")
1078
1079 if self.config.doDefect is True:
1080 if "defects" in inputs and inputs['defects'] is not None:
1081 # defects is loaded as a BaseCatalog with columns
1082 # x0, y0, width, height. Masking expects a list of defects
1083 # defined by their bounding box
1084 if not isinstance(inputs["defects"], Defects):
1085 inputs["defects"] = Defects.fromTable(inputs["defects"])
1086
1087 # Load the correct style of brighter-fatter kernel, and repack
1088 # the information as a numpy array.
1089 if self.config.doBrighterFatter:
1090 brighterFatterKernel = inputs.pop('newBFKernel', None)
1091 if brighterFatterKernel is None:
1092 brighterFatterKernel = inputs.get('bfKernel', None)
1093
1094 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1095 # This is a ISR calib kernel
1096 detName = detector.getName()
1097 level = brighterFatterKernel.level
1098
1099 # This is expected to be a dictionary of amp-wise gains.
1100 inputs['bfGains'] = brighterFatterKernel.gain
1101 if self.config.brighterFatterLevel == 'DETECTOR':
1102 if level == 'DETECTOR':
1103 if detName in brighterFatterKernel.detKernels:
1104 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1105 else:
1106 raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1107 elif level == 'AMP':
1108 self.log.warning("Making DETECTOR level kernel from AMP based brighter "
1109 "fatter kernels.")
1110 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1111 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1112 elif self.config.brighterFatterLevel == 'AMP':
1113 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1114
1115 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
1116 expId = inputs['ccdExposure'].info.id
1117 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
1118 expId=expId,
1119 assembler=self.assembleCcd
1120 if self.config.doAssembleIsrExposures else None)
1121 else:
1122 inputs['fringes'] = pipeBase.Struct(fringes=None)
1123
1124 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
1125 if 'strayLightData' not in inputs:
1126 inputs['strayLightData'] = None
1127
1128 outputs = self.run(**inputs)
1129 butlerQC.put(outputs, outputRefs)
1130
1131 @timeMethod
1132 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1133 crosstalk=None, crosstalkSources=None,
1134 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1135 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1136 sensorTransmission=None, atmosphereTransmission=None,
1137 detectorNum=None, strayLightData=None, illumMaskedImage=None,
1138 deferredCharge=None,
1139 ):
1140 """Perform instrument signature removal on an exposure.
1141
1142 Steps included in the ISR processing, in order performed, are:
1143 - saturation and suspect pixel masking
1144 - overscan subtraction
1145 - CCD assembly of individual amplifiers
1146 - bias subtraction
1147 - variance image construction
1148 - linearization of non-linear response
1149 - crosstalk masking
1150 - brighter-fatter correction
1151 - dark subtraction
1152 - fringe correction
1153 - stray light subtraction
1154 - flat correction
1155 - masking of known defects and camera specific features
1156 - vignette calculation
1157 - appending transmission curve and distortion model
1158
1159 Parameters
1160 ----------
1161 ccdExposure : `lsst.afw.image.Exposure`
1162 The raw exposure that is to be run through ISR. The
1163 exposure is modified by this method.
1164 camera : `lsst.afw.cameraGeom.Camera`, optional
1165 The camera geometry for this exposure. Required if
1166 one or more of ``ccdExposure``, ``bias``, ``dark``, or
1167 ``flat`` does not have an associated detector.
1168 bias : `lsst.afw.image.Exposure`, optional
1169 Bias calibration frame.
1170 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1171 Functor for linearization.
1172 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1173 Calibration for crosstalk.
1174 crosstalkSources : `list`, optional
1175 List of possible crosstalk sources.
1176 dark : `lsst.afw.image.Exposure`, optional
1177 Dark calibration frame.
1178 flat : `lsst.afw.image.Exposure`, optional
1179 Flat calibration frame.
1181 Photon transfer curve dataset, with, e.g., gains
1182 and read noise.
1183 bfKernel : `numpy.ndarray`, optional
1184 Brighter-fatter kernel.
1185 bfGains : `dict` of `float`, optional
1186 Gains used to override the detector's nominal gains for the
1187 brighter-fatter correction. A dict keyed by amplifier name for
1188 the detector in question.
1189 defects : `lsst.ip.isr.Defects`, optional
1190 List of defects.
1191 fringes : `lsst.pipe.base.Struct`, optional
1192 Struct containing the fringe correction data, with
1193 elements:
1194 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1195 - ``seed``: random seed derived from the ccdExposureId for random
1196 number generator (`uint32`)
1197 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1198 A ``TransmissionCurve`` that represents the throughput of the,
1199 optics, to be evaluated in focal-plane coordinates.
1200 filterTransmission : `lsst.afw.image.TransmissionCurve`
1201 A ``TransmissionCurve`` that represents the throughput of the
1202 filter itself, to be evaluated in focal-plane coordinates.
1203 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1204 A ``TransmissionCurve`` that represents the throughput of the
1205 sensor itself, to be evaluated in post-assembly trimmed detector
1206 coordinates.
1207 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1208 A ``TransmissionCurve`` that represents the throughput of the
1209 atmosphere, assumed to be spatially constant.
1210 detectorNum : `int`, optional
1211 The integer number for the detector to process.
1212 strayLightData : `object`, optional
1213 Opaque object containing calibration information for stray-light
1214 correction. If `None`, no correction will be performed.
1215 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1216 Illumination correction image.
1217
1218 Returns
1219 -------
1220 result : `lsst.pipe.base.Struct`
1221 Result struct with component:
1222 - ``exposure`` : `afw.image.Exposure`
1223 The fully ISR corrected exposure.
1224 - ``outputExposure`` : `afw.image.Exposure`
1225 An alias for `exposure`
1226 - ``ossThumb`` : `numpy.ndarray`
1227 Thumbnail image of the exposure after overscan subtraction.
1228 - ``flattenedThumb`` : `numpy.ndarray`
1229 Thumbnail image of the exposure after flat-field correction.
1230 - ``outputStatistics`` : ``
1231 Values of the additional statistics calculated.
1232
1233 Raises
1234 ------
1235 RuntimeError
1236 Raised if a configuration option is set to True, but the
1237 required calibration data has not been specified.
1238
1239 Notes
1240 -----
1241 The current processed exposure can be viewed by setting the
1242 appropriate lsstDebug entries in the `debug.display`
1243 dictionary. The names of these entries correspond to some of
1244 the IsrTaskConfig Boolean options, with the value denoting the
1245 frame to use. The exposure is shown inside the matching
1246 option check and after the processing of that step has
1247 finished. The steps with debug points are:
1248
1249 doAssembleCcd
1250 doBias
1251 doCrosstalk
1252 doBrighterFatter
1253 doDark
1254 doFringe
1255 doStrayLight
1256 doFlat
1257
1258 In addition, setting the "postISRCCD" entry displays the
1259 exposure after all ISR processing has finished.
1260
1261 """
1262
1263 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1264 bias = self.ensureExposure(bias, camera, detectorNum)
1265 dark = self.ensureExposure(dark, camera, detectorNum)
1266 flat = self.ensureExposure(flat, camera, detectorNum)
1267
1268 ccd = ccdExposure.getDetector()
1269 filterLabel = ccdExposure.getFilter()
1270 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1271
1272 if not ccd:
1273 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1274 ccd = [FakeAmp(ccdExposure, self.config)]
1275
1276 # Validate Input
1277 if self.config.doBias and bias is None:
1278 raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1279 if self.doLinearize(ccd) and linearizer is None:
1280 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1281 if self.config.doBrighterFatter and bfKernel is None:
1282 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1283 if self.config.doDark and dark is None:
1284 raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1285 if self.config.doFlat and flat is None:
1286 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1287 if self.config.doDefect and defects is None:
1288 raise RuntimeError("Must supply defects if config.doDefect=True.")
1289 if (self.config.doFringe and physicalFilter in self.fringe.config.filters
1290 and fringes.fringes is None):
1291 # The `fringes` object needs to be a pipeBase.Struct, as
1292 # we use it as a `dict` for the parameters of
1293 # `FringeTask.run()`. The `fringes.fringes` `list` may
1294 # not be `None` if `doFringe=True`. Otherwise, raise.
1295 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1296 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters
1297 and illumMaskedImage is None):
1298 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1299 if (self.config.doDeferredCharge and deferredCharge is None):
1300 raise RuntimeError("Must supply a deferred charge calibration if config.doDeferredCharge=True.")
1301
1302 # Begin ISR processing.
1303 if self.config.doConvertIntToFloat:
1304 self.log.info("Converting exposure to floating point values.")
1305 ccdExposure = self.convertIntToFloat(ccdExposure)
1306
1307 if self.config.doBias and self.config.doBiasBeforeOverscan:
1308 self.log.info("Applying bias correction.")
1309 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1310 trimToFit=self.config.doTrimToMatchCalib)
1311 self.debugView(ccdExposure, "doBias")
1312
1313 # Amplifier level processing.
1314 overscans = []
1315 for amp in ccd:
1316 # if ccdExposure is one amp,
1317 # check for coverage to prevent performing ops multiple times
1318 if ccdExposure.getBBox().contains(amp.getBBox()):
1319 # Check for fully masked bad amplifiers,
1320 # and generate masks for SUSPECT and SATURATED values.
1321 badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1322
1323 if self.config.doOverscan and not badAmp:
1324 # Overscan correction on amp-by-amp basis.
1325 overscanResults = self.overscanCorrection(ccdExposure, amp)
1326 self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1327 if overscanResults is not None and \
1328 self.config.qa is not None and self.config.qa.saveStats is True:
1329 if isinstance(overscanResults.overscanFit, float):
1330 qaMedian = overscanResults.overscanFit
1331 qaStdev = float("NaN")
1332 else:
1333 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1334 afwMath.MEDIAN | afwMath.STDEVCLIP)
1335 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1336 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1337
1338 self.metadata[f"FIT MEDIAN {amp.getName()}"] = qaMedian
1339 self.metadata[f"FIT STDEV {amp.getName()}"] = qaStdev
1340 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1341 amp.getName(), qaMedian, qaStdev)
1342
1343 # Residuals after overscan correction
1344 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1345 afwMath.MEDIAN | afwMath.STDEVCLIP)
1346 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1347 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1348
1349 self.metadata[f"RESIDUAL MEDIAN {amp.getName()}"] = qaMedianAfter
1350 self.metadata[f"RESIDUAL STDEV {amp.getName()}"] = qaStdevAfter
1351 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f",
1352 amp.getName(), qaMedianAfter, qaStdevAfter)
1353
1354 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1355 else:
1356 if badAmp:
1357 self.log.warning("Amplifier %s is bad.", amp.getName())
1358 overscanResults = None
1359
1360 overscans.append(overscanResults if overscanResults is not None else None)
1361 else:
1362 self.log.info("Skipped OSCAN for %s.", amp.getName())
1363
1364 if self.config.doDeferredCharge:
1365 self.log.info("Applying deferred charge/CTI correction.")
1366 self.deferredChargeCorrection.run(ccdExposure, deferredCharge)
1367 self.debugView(ccdExposure, "doDeferredCharge")
1368
1369 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1370 self.log.info("Applying crosstalk correction.")
1371 self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1372 crosstalkSources=crosstalkSources, camera=camera)
1373 self.debugView(ccdExposure, "doCrosstalk")
1374
1375 if self.config.doAssembleCcd:
1376 self.log.info("Assembling CCD from amplifiers.")
1377 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1378
1379 if self.config.expectWcs and not ccdExposure.getWcs():
1380 self.log.warning("No WCS found in input exposure.")
1381 self.debugView(ccdExposure, "doAssembleCcd")
1382
1383 ossThumb = None
1384 if self.config.qa.doThumbnailOss:
1385 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1386
1387 if self.config.doBias and not self.config.doBiasBeforeOverscan:
1388 self.log.info("Applying bias correction.")
1389 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1390 trimToFit=self.config.doTrimToMatchCalib)
1391 self.debugView(ccdExposure, "doBias")
1392
1393 if self.config.doVariance:
1394 for amp, overscanResults in zip(ccd, overscans):
1395 if ccdExposure.getBBox().contains(amp.getBBox()):
1396 self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1397 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1398 if overscanResults is not None:
1399 self.updateVariance(ampExposure, amp,
1400 overscanImage=overscanResults.overscanImage,
1401 ptcDataset=ptc)
1402 else:
1403 self.updateVariance(ampExposure, amp,
1404 overscanImage=None,
1405 ptcDataset=ptc)
1406 if self.config.qa is not None and self.config.qa.saveStats is True:
1407 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1408 afwMath.MEDIAN | afwMath.STDEVCLIP)
1409 self.metadata[f"ISR VARIANCE {amp.getName()} MEDIAN"] = \
1410 qaStats.getValue(afwMath.MEDIAN)
1411 self.metadata[f"ISR VARIANCE {amp.getName()} STDEV"] = \
1412 qaStats.getValue(afwMath.STDEVCLIP)
1413 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1414 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1415 qaStats.getValue(afwMath.STDEVCLIP))
1416 if self.config.maskNegativeVariance:
1417 self.maskNegativeVariance(ccdExposure)
1418
1419 if self.doLinearize(ccd):
1420 self.log.info("Applying linearizer.")
1421 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1422 detector=ccd, log=self.log)
1423
1424 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1425 self.log.info("Applying crosstalk correction.")
1426 self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1427 crosstalkSources=crosstalkSources, isTrimmed=True)
1428 self.debugView(ccdExposure, "doCrosstalk")
1429
1430 # Masking block. Optionally mask known defects, NAN/inf pixels,
1431 # widen trails, and do anything else the camera needs. Saturated and
1432 # suspect pixels have already been masked.
1433 if self.config.doDefect:
1434 self.log.info("Masking defects.")
1435 self.maskDefect(ccdExposure, defects)
1436
1437 if self.config.numEdgeSuspect > 0:
1438 self.log.info("Masking edges as SUSPECT.")
1439 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1440 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
1441
1442 if self.config.doNanMasking:
1443 self.log.info("Masking non-finite (NAN, inf) value pixels.")
1444 self.maskNan(ccdExposure)
1445
1446 if self.config.doWidenSaturationTrails:
1447 self.log.info("Widening saturation trails.")
1448 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1449
1450 if self.config.doCameraSpecificMasking:
1451 self.log.info("Masking regions for camera specific reasons.")
1452 self.masking.run(ccdExposure)
1453
1454 if self.config.doBrighterFatter:
1455 # We need to apply flats and darks before we can interpolate, and
1456 # we need to interpolate before we do B-F, but we do B-F without
1457 # the flats and darks applied so we can work in units of electrons
1458 # or holes. This context manager applies and then removes the darks
1459 # and flats.
1460 #
1461 # We also do not want to interpolate values here, so operate on
1462 # temporary images so we can apply only the BF-correction and roll
1463 # back the interpolation.
1464 interpExp = ccdExposure.clone()
1465 with self.flatContext(interpExp, flat, dark):
1466 isrFunctions.interpolateFromMask(
1467 maskedImage=interpExp.getMaskedImage(),
1468 fwhm=self.config.fwhm,
1469 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1470 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1471 )
1472 bfExp = interpExp.clone()
1473
1474 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.",
1475 type(bfKernel), type(bfGains))
1476 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1477 self.config.brighterFatterMaxIter,
1478 self.config.brighterFatterThreshold,
1479 self.config.brighterFatterApplyGain,
1480 bfGains)
1481 if bfResults[1] == self.config.brighterFatterMaxIter:
1482 self.log.warning("Brighter-fatter correction did not converge, final difference %f.",
1483 bfResults[0])
1484 else:
1485 self.log.info("Finished brighter-fatter correction in %d iterations.",
1486 bfResults[1])
1487 image = ccdExposure.getMaskedImage().getImage()
1488 bfCorr = bfExp.getMaskedImage().getImage()
1489 bfCorr -= interpExp.getMaskedImage().getImage()
1490 image += bfCorr
1491
1492 # Applying the brighter-fatter correction applies a
1493 # convolution to the science image. At the edges this
1494 # convolution may not have sufficient valid pixels to
1495 # produce a valid correction. Mark pixels within the size
1496 # of the brighter-fatter kernel as EDGE to warn of this
1497 # fact.
1498 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1499 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1500 maskPlane="EDGE")
1501
1502 if self.config.brighterFatterMaskGrowSize > 0:
1503 self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1504 for maskPlane in self.config.brighterFatterMaskListToInterpolate:
1505 isrFunctions.growMasks(ccdExposure.getMask(),
1506 radius=self.config.brighterFatterMaskGrowSize,
1507 maskNameList=maskPlane,
1508 maskValue=maskPlane)
1509
1510 self.debugView(ccdExposure, "doBrighterFatter")
1511
1512 if self.config.doDark:
1513 self.log.info("Applying dark correction.")
1514 self.darkCorrection(ccdExposure, dark)
1515 self.debugView(ccdExposure, "doDark")
1516
1517 if self.config.doFringe and not self.config.fringeAfterFlat:
1518 self.log.info("Applying fringe correction before flat.")
1519 self.fringe.run(ccdExposure, **fringes.getDict())
1520 self.debugView(ccdExposure, "doFringe")
1521
1522 if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1523 self.log.info("Checking strayLight correction.")
1524 self.strayLight.run(ccdExposure, strayLightData)
1525 self.debugView(ccdExposure, "doStrayLight")
1526
1527 if self.config.doFlat:
1528 self.log.info("Applying flat correction.")
1529 self.flatCorrection(ccdExposure, flat)
1530 self.debugView(ccdExposure, "doFlat")
1531
1532 if self.config.doApplyGains:
1533 self.log.info("Applying gain correction instead of flat.")
1534 if self.config.usePtcGains:
1535 self.log.info("Using gains from the Photon Transfer Curve.")
1536 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1537 ptcGains=ptc.gain)
1538 else:
1539 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1540
1541 if self.config.doFringe and self.config.fringeAfterFlat:
1542 self.log.info("Applying fringe correction after flat.")
1543 self.fringe.run(ccdExposure, **fringes.getDict())
1544
1545 if self.config.doVignette:
1546 if self.config.doMaskVignettePolygon:
1547 self.log.info("Constructing, attaching, and masking vignette polygon.")
1548 else:
1549 self.log.info("Constructing and attaching vignette polygon.")
1550 self.vignettePolygon = self.vignette.run(
1551 exposure=ccdExposure, doUpdateMask=self.config.doMaskVignettePolygon,
1552 vignetteValue=self.config.vignetteValue, log=self.log)
1553
1554 if self.config.doAttachTransmissionCurve:
1555 self.log.info("Adding transmission curves.")
1556 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1557 filterTransmission=filterTransmission,
1558 sensorTransmission=sensorTransmission,
1559 atmosphereTransmission=atmosphereTransmission)
1560
1561 flattenedThumb = None
1562 if self.config.qa.doThumbnailFlattened:
1563 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1564
1565 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters:
1566 self.log.info("Performing illumination correction.")
1567 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1568 illumMaskedImage, illumScale=self.config.illumScale,
1569 trimToFit=self.config.doTrimToMatchCalib)
1570
1571 preInterpExp = None
1572 if self.config.doSaveInterpPixels:
1573 preInterpExp = ccdExposure.clone()
1574
1575 # Reset and interpolate bad pixels.
1576 #
1577 # Large contiguous bad regions (which should have the BAD mask
1578 # bit set) should have their values set to the image median.
1579 # This group should include defects and bad amplifiers. As the
1580 # area covered by these defects are large, there's little
1581 # reason to expect that interpolation would provide a more
1582 # useful value.
1583 #
1584 # Smaller defects can be safely interpolated after the larger
1585 # regions have had their pixel values reset. This ensures
1586 # that the remaining defects adjacent to bad amplifiers (as an
1587 # example) do not attempt to interpolate extreme values.
1588 if self.config.doSetBadRegions:
1589 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1590 if badPixelCount > 0:
1591 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1592
1593 if self.config.doInterpolate:
1594 self.log.info("Interpolating masked pixels.")
1595 isrFunctions.interpolateFromMask(
1596 maskedImage=ccdExposure.getMaskedImage(),
1597 fwhm=self.config.fwhm,
1598 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1599 maskNameList=list(self.config.maskListToInterpolate)
1600 )
1601
1602 self.roughZeroPoint(ccdExposure)
1603
1604 # correct for amp offsets within the CCD
1605 if self.config.doAmpOffset:
1606 self.log.info("Correcting amp offsets.")
1607 self.ampOffset.run(ccdExposure)
1608
1609 if self.config.doMeasureBackground:
1610 self.log.info("Measuring background level.")
1611 self.measureBackground(ccdExposure, self.config.qa)
1612
1613 if self.config.qa is not None and self.config.qa.saveStats is True:
1614 for amp in ccd:
1615 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1616 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1617 afwMath.MEDIAN | afwMath.STDEVCLIP)
1618 self.metadata[f"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN)
1619 self.metadata[f"ISR BACKGROUND {amp.getName()} STDEV"] = \
1620 qaStats.getValue(afwMath.STDEVCLIP)
1621 self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1622 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1623 qaStats.getValue(afwMath.STDEVCLIP))
1624
1625 # calculate additional statistics.
1626 outputStatistics = None
1627 if self.config.doCalculateStatistics:
1628 outputStatistics = self.isrStats.run(ccdExposure, overscanResults=overscans,
1629 ptc=ptc).results
1630
1631 self.debugView(ccdExposure, "postISRCCD")
1632
1633 return pipeBase.Struct(
1634 exposure=ccdExposure,
1635 ossThumb=ossThumb,
1636 flattenedThumb=flattenedThumb,
1637
1638 preInterpExposure=preInterpExp,
1639 outputExposure=ccdExposure,
1640 outputOssThumbnail=ossThumb,
1641 outputFlattenedThumbnail=flattenedThumb,
1642 outputStatistics=outputStatistics,
1643 )
1644
1645 def ensureExposure(self, inputExp, camera=None, detectorNum=None):
1646 """Ensure that the data returned by Butler is a fully constructed exp.
1647
1648 ISR requires exposure-level image data for historical reasons, so if we
1649 did not recieve that from Butler, construct it from what we have,
1650 modifying the input in place.
1651
1652 Parameters
1653 ----------
1654 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`,
1655 or `lsst.afw.image.ImageF`
1656 The input data structure obtained from Butler.
1657 camera : `lsst.afw.cameraGeom.camera`, optional
1658 The camera associated with the image. Used to find the appropriate
1659 detector if detector is not already set.
1660 detectorNum : `int`, optional
1661 The detector in the camera to attach, if the detector is not
1662 already set.
1663
1664 Returns
1665 -------
1666 inputExp : `lsst.afw.image.Exposure`
1667 The re-constructed exposure, with appropriate detector parameters.
1668
1669 Raises
1670 ------
1671 TypeError
1672 Raised if the input data cannot be used to construct an exposure.
1673 """
1674 if isinstance(inputExp, afwImage.DecoratedImageU):
1675 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1676 elif isinstance(inputExp, afwImage.ImageF):
1677 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1678 elif isinstance(inputExp, afwImage.MaskedImageF):
1679 inputExp = afwImage.makeExposure(inputExp)
1680 elif isinstance(inputExp, afwImage.Exposure):
1681 pass
1682 elif inputExp is None:
1683 # Assume this will be caught by the setup if it is a problem.
1684 return inputExp
1685 else:
1686 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1687 (type(inputExp), ))
1688
1689 if inputExp.getDetector() is None:
1690 if camera is None or detectorNum is None:
1691 raise RuntimeError('Must supply both a camera and detector number when using exposures '
1692 'without a detector set.')
1693 inputExp.setDetector(camera[detectorNum])
1694
1695 return inputExp
1696
1697 def convertIntToFloat(self, exposure):
1698 """Convert exposure image from uint16 to float.
1699
1700 If the exposure does not need to be converted, the input is
1701 immediately returned. For exposures that are converted to use
1702 floating point pixels, the variance is set to unity and the
1703 mask to zero.
1704
1705 Parameters
1706 ----------
1707 exposure : `lsst.afw.image.Exposure`
1708 The raw exposure to be converted.
1709
1710 Returns
1711 -------
1712 newexposure : `lsst.afw.image.Exposure`
1713 The input ``exposure``, converted to floating point pixels.
1714
1715 Raises
1716 ------
1717 RuntimeError
1718 Raised if the exposure type cannot be converted to float.
1719
1720 """
1721 if isinstance(exposure, afwImage.ExposureF):
1722 # Nothing to be done
1723 self.log.debug("Exposure already of type float.")
1724 return exposure
1725 if not hasattr(exposure, "convertF"):
1726 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1727
1728 newexposure = exposure.convertF()
1729 newexposure.variance[:] = 1
1730 newexposure.mask[:] = 0x0
1731
1732 return newexposure
1733
1734 def maskAmplifier(self, ccdExposure, amp, defects):
1735 """Identify bad amplifiers, saturated and suspect pixels.
1736
1737 Parameters
1738 ----------
1739 ccdExposure : `lsst.afw.image.Exposure`
1740 Input exposure to be masked.
1742 Catalog of parameters defining the amplifier on this
1743 exposure to mask.
1744 defects : `lsst.ip.isr.Defects`
1745 List of defects. Used to determine if the entire
1746 amplifier is bad.
1747
1748 Returns
1749 -------
1750 badAmp : `Bool`
1751 If this is true, the entire amplifier area is covered by
1752 defects and unusable.
1753
1754 """
1755 maskedImage = ccdExposure.getMaskedImage()
1756
1757 badAmp = False
1758
1759 # Check if entire amp region is defined as a defect
1760 # NB: need to use amp.getBBox() for correct comparison with current
1761 # defects definition.
1762 if defects is not None:
1763 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1764
1765 # In the case of a bad amp, we will set mask to "BAD"
1766 # (here use amp.getRawBBox() for correct association with pixels in
1767 # current ccdExposure).
1768 if badAmp:
1769 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1770 afwImage.PARENT)
1771 maskView = dataView.getMask()
1772 maskView |= maskView.getPlaneBitMask("BAD")
1773 del maskView
1774 return badAmp
1775
1776 # Mask remaining defects after assembleCcd() to allow for defects that
1777 # cross amplifier boundaries. Saturation and suspect pixels can be
1778 # masked now, though.
1779 limits = dict()
1780 if self.config.doSaturation and not badAmp:
1781 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1782 if self.config.doSuspect and not badAmp:
1783 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1784 if math.isfinite(self.config.saturation):
1785 limits.update({self.config.saturatedMaskName: self.config.saturation})
1786
1787 for maskName, maskThreshold in limits.items():
1788 if not math.isnan(maskThreshold):
1789 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1790 isrFunctions.makeThresholdMask(
1791 maskedImage=dataView,
1792 threshold=maskThreshold,
1793 growFootprints=0,
1794 maskName=maskName
1795 )
1796
1797 # Determine if we've fully masked this amplifier with SUSPECT and
1798 # SAT pixels.
1799 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1800 afwImage.PARENT)
1801 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1802 self.config.suspectMaskName])
1803 if numpy.all(maskView.getArray() & maskVal > 0):
1804 badAmp = True
1805 maskView |= maskView.getPlaneBitMask("BAD")
1806
1807 return badAmp
1808
1809 def overscanCorrection(self, ccdExposure, amp):
1810 """Apply overscan correction in place.
1811
1812 This method does initial pixel rejection of the overscan
1813 region. The overscan can also be optionally segmented to
1814 allow for discontinuous overscan responses to be fit
1815 separately. The actual overscan subtraction is performed by
1816 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1817 which is called here after the amplifier is preprocessed.
1818
1819 Parameters
1820 ----------
1821 ccdExposure : `lsst.afw.image.Exposure`
1822 Exposure to have overscan correction performed.
1823 amp : `lsst.afw.cameraGeom.Amplifer`
1824 The amplifier to consider while correcting the overscan.
1825
1826 Returns
1827 -------
1828 overscanResults : `lsst.pipe.base.Struct`
1829 Result struct with components:
1830 - ``imageFit`` : scalar or `lsst.afw.image.Image`
1831 Value or fit subtracted from the amplifier image data.
1832 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1833 Value or fit subtracted from the overscan image data.
1834 - ``overscanImage`` : `lsst.afw.image.Image`
1835 Image of the overscan region with the overscan
1836 correction applied. This quantity is used to estimate
1837 the amplifier read noise empirically.
1838
1839 Raises
1840 ------
1841 RuntimeError
1842 Raised if the ``amp`` does not contain raw pixel information.
1843
1844 See Also
1845 --------
1846 lsst.ip.isr.isrFunctions.overscanCorrection
1847 """
1848 if amp.getRawHorizontalOverscanBBox().isEmpty():
1849 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1850 return None
1851
1852 statControl = afwMath.StatisticsControl()
1853 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1854
1855 # Determine the bounding boxes
1856 dataBBox = amp.getRawDataBBox()
1857 oscanBBox = amp.getRawHorizontalOverscanBBox()
1858 dx0 = 0
1859 dx1 = 0
1860
1861 prescanBBox = amp.getRawPrescanBBox()
1862 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1863 dx0 += self.config.overscanNumLeadingColumnsToSkip
1864 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1865 else:
1866 dx0 += self.config.overscanNumTrailingColumnsToSkip
1867 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1868
1869 # Determine if we need to work on subregions of the amplifier
1870 # and overscan.
1871 imageBBoxes = []
1872 overscanBBoxes = []
1873
1874 if ((self.config.overscanBiasJump
1875 and self.config.overscanBiasJumpLocation)
1876 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
1877 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1878 self.config.overscanBiasJumpDevices)):
1879 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
1880 yLower = self.config.overscanBiasJumpLocation
1881 yUpper = dataBBox.getHeight() - yLower
1882 else:
1883 yUpper = self.config.overscanBiasJumpLocation
1884 yLower = dataBBox.getHeight() - yUpper
1885
1886 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1887 lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1888 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1889 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1890 yLower)))
1891
1892 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1893 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1894 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1895 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1896 yUpper)))
1897 else:
1898 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1899 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1900 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1901 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1902 oscanBBox.getHeight())))
1903
1904 # Perform overscan correction on subregions, ensuring saturated
1905 # pixels are masked.
1906 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1907 ampImage = ccdExposure.maskedImage[imageBBox]
1908 overscanImage = ccdExposure.maskedImage[overscanBBox]
1909
1910 overscanArray = overscanImage.image.array
1911 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1912 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1913 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1914
1915 statControl = afwMath.StatisticsControl()
1916 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1917
1918 overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp)
1919
1920 # If we trimmed columns, we need to restore them.
1921 if dx0 != 0 or dx1 != 0:
1922 fullOverscan = ccdExposure.maskedImage[oscanBBox]
1923 overscanVector = overscanResults.overscanFit.array[:, 0]
1924 overscanModel = afwImage.ImageF(fullOverscan.getDimensions())
1925 overscanModel.array[:, :] = 0.0
1926 overscanModel.array[:, 0:dx0] = overscanVector[:, numpy.newaxis]
1927 overscanModel.array[:, dx1:] = overscanVector[:, numpy.newaxis]
1928 fullOverscanImage = fullOverscan.getImage()
1929 fullOverscanImage -= overscanModel
1930 overscanResults = pipeBase.Struct(imageFit=overscanResults.imageFit,
1931 overscanFit=overscanModel,
1932 overscanImage=fullOverscan,
1933 edgeMask=overscanResults.edgeMask)
1934
1935 # Measure average overscan levels and record them in the metadata.
1936 levelStat = afwMath.MEDIAN
1937 sigmaStat = afwMath.STDEVCLIP
1938
1939 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1940 self.config.qa.flatness.nIter)
1941 metadata = ccdExposure.getMetadata()
1942 ampNum = amp.getName()
1943 # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1944 if isinstance(overscanResults.overscanFit, float):
1945 metadata[f"ISR_OSCAN_LEVEL{ampNum}"] = overscanResults.overscanFit
1946 metadata[f"ISR_OSCAN_SIGMA{ampNum}"] = 0.0
1947 else:
1948 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1949 metadata[f"ISR_OSCAN_LEVEL{ampNum}"] = stats.getValue(levelStat)
1950 metadata[f"ISR_OSCAN_SIGMA%{ampNum}"] = stats.getValue(sigmaStat)
1951
1952 return overscanResults
1953
1954 def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None):
1955 """Set the variance plane using the gain and read noise
1956
1957 The read noise is calculated from the ``overscanImage`` if the
1958 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1959 the value from the amplifier data is used.
1960
1961 Parameters
1962 ----------
1963 ampExposure : `lsst.afw.image.Exposure`
1964 Exposure to process.
1965 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
1966 Amplifier detector data.
1967 overscanImage : `lsst.afw.image.MaskedImage`, optional.
1968 Image of overscan, required only for empirical read noise.
1969 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
1970 PTC dataset containing the gains and read noise.
1971
1972
1973 Raises
1974 ------
1975 RuntimeError
1976 Raised if either ``usePtcGains`` of ``usePtcReadNoise``
1977 are ``True``, but ptcDataset is not provided.
1978
1979 Raised if ```doEmpiricalReadNoise`` is ``True`` but
1980 ``overscanImage`` is ``None``.
1981
1982 See also
1983 --------
1984 lsst.ip.isr.isrFunctions.updateVariance
1985 """
1986 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
1987 if self.config.usePtcGains:
1988 if ptcDataset is None:
1989 raise RuntimeError("No ptcDataset provided to use PTC gains.")
1990 else:
1991 gain = ptcDataset.gain[amp.getName()]
1992 self.log.info("Using gain from Photon Transfer Curve.")
1993 else:
1994 gain = amp.getGain()
1995
1996 if math.isnan(gain):
1997 gain = 1.0
1998 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
1999 elif gain <= 0:
2000 patchedGain = 1.0
2001 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.",
2002 amp.getName(), gain, patchedGain)
2003 gain = patchedGain
2004
2005 if self.config.doEmpiricalReadNoise and overscanImage is None:
2006 raise RuntimeError("Overscan is none for EmpiricalReadNoise.")
2007
2008 if self.config.doEmpiricalReadNoise and overscanImage is not None:
2009 stats = afwMath.StatisticsControl()
2010 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2011 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2012 self.log.info("Calculated empirical read noise for amp %s: %f.",
2013 amp.getName(), readNoise)
2014 elif self.config.usePtcReadNoise:
2015 if ptcDataset is None:
2016 raise RuntimeError("No ptcDataset provided to use PTC readnoise.")
2017 else:
2018 readNoise = ptcDataset.noise[amp.getName()]
2019 self.log.info("Using read noise from Photon Transfer Curve.")
2020 else:
2021 readNoise = amp.getReadNoise()
2022
2023 isrFunctions.updateVariance(
2024 maskedImage=ampExposure.getMaskedImage(),
2025 gain=gain,
2026 readNoise=readNoise,
2027 )
2028
2029 def maskNegativeVariance(self, exposure):
2030 """Identify and mask pixels with negative variance values.
2031
2032 Parameters
2033 ----------
2034 exposure : `lsst.afw.image.Exposure`
2035 Exposure to process.
2036
2037 See Also
2038 --------
2039 lsst.ip.isr.isrFunctions.updateVariance
2040 """
2041 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
2042 bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
2043 exposure.mask.array[bad] |= maskPlane
2044
2045 def darkCorrection(self, exposure, darkExposure, invert=False):
2046 """Apply dark correction in place.
2047
2048 Parameters
2049 ----------
2050 exposure : `lsst.afw.image.Exposure`
2051 Exposure to process.
2052 darkExposure : `lsst.afw.image.Exposure`
2053 Dark exposure of the same size as ``exposure``.
2054 invert : `Bool`, optional
2055 If True, re-add the dark to an already corrected image.
2056
2057 Raises
2058 ------
2059 RuntimeError
2060 Raised if either ``exposure`` or ``darkExposure`` do not
2061 have their dark time defined.
2062
2063 See Also
2064 --------
2065 lsst.ip.isr.isrFunctions.darkCorrection
2066 """
2067 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2068 if math.isnan(expScale):
2069 raise RuntimeError("Exposure darktime is NAN.")
2070 if darkExposure.getInfo().getVisitInfo() is not None \
2071 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2072 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2073 else:
2074 # DM-17444: darkExposure.getInfo.getVisitInfo() is None
2075 # so getDarkTime() does not exist.
2076 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2077 darkScale = 1.0
2078
2079 isrFunctions.darkCorrection(
2080 maskedImage=exposure.getMaskedImage(),
2081 darkMaskedImage=darkExposure.getMaskedImage(),
2082 expScale=expScale,
2083 darkScale=darkScale,
2084 invert=invert,
2085 trimToFit=self.config.doTrimToMatchCalib
2086 )
2087
2088 def doLinearize(self, detector):
2089 """Check if linearization is needed for the detector cameraGeom.
2090
2091 Checks config.doLinearize and the linearity type of the first
2092 amplifier.
2093
2094 Parameters
2095 ----------
2096 detector : `lsst.afw.cameraGeom.Detector`
2097 Detector to get linearity type from.
2098
2099 Returns
2100 -------
2101 doLinearize : `Bool`
2102 If True, linearization should be performed.
2103 """
2104 return self.config.doLinearize and \
2105 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2106
2107 def flatCorrection(self, exposure, flatExposure, invert=False):
2108 """Apply flat correction in place.
2109
2110 Parameters
2111 ----------
2112 exposure : `lsst.afw.image.Exposure`
2113 Exposure to process.
2114 flatExposure : `lsst.afw.image.Exposure`
2115 Flat exposure of the same size as ``exposure``.
2116 invert : `Bool`, optional
2117 If True, unflatten an already flattened image.
2118
2119 See Also
2120 --------
2121 lsst.ip.isr.isrFunctions.flatCorrection
2122 """
2123 isrFunctions.flatCorrection(
2124 maskedImage=exposure.getMaskedImage(),
2125 flatMaskedImage=flatExposure.getMaskedImage(),
2126 scalingType=self.config.flatScalingType,
2127 userScale=self.config.flatUserScale,
2128 invert=invert,
2129 trimToFit=self.config.doTrimToMatchCalib
2130 )
2131
2132 def saturationDetection(self, exposure, amp):
2133 """Detect and mask saturated pixels in config.saturatedMaskName.
2134
2135 Parameters
2136 ----------
2137 exposure : `lsst.afw.image.Exposure`
2138 Exposure to process. Only the amplifier DataSec is processed.
2140 Amplifier detector data.
2141
2142 See Also
2143 --------
2144 lsst.ip.isr.isrFunctions.makeThresholdMask
2145 """
2146 if not math.isnan(amp.getSaturation()):
2147 maskedImage = exposure.getMaskedImage()
2148 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2149 isrFunctions.makeThresholdMask(
2150 maskedImage=dataView,
2151 threshold=amp.getSaturation(),
2152 growFootprints=0,
2153 maskName=self.config.saturatedMaskName,
2154 )
2155
2156 def saturationInterpolation(self, exposure):
2157 """Interpolate over saturated pixels, in place.
2158
2159 This method should be called after `saturationDetection`, to
2160 ensure that the saturated pixels have been identified in the
2161 SAT mask. It should also be called after `assembleCcd`, since
2162 saturated regions may cross amplifier boundaries.
2163
2164 Parameters
2165 ----------
2166 exposure : `lsst.afw.image.Exposure`
2167 Exposure to process.
2168
2169 See Also
2170 --------
2171 lsst.ip.isr.isrTask.saturationDetection
2172 lsst.ip.isr.isrFunctions.interpolateFromMask
2173 """
2174 isrFunctions.interpolateFromMask(
2175 maskedImage=exposure.getMaskedImage(),
2176 fwhm=self.config.fwhm,
2177 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2178 maskNameList=list(self.config.saturatedMaskName),
2179 )
2180
2181 def suspectDetection(self, exposure, amp):
2182 """Detect and mask suspect pixels in config.suspectMaskName.
2183
2184 Parameters
2185 ----------
2186 exposure : `lsst.afw.image.Exposure`
2187 Exposure to process. Only the amplifier DataSec is processed.
2189 Amplifier detector data.
2190
2191 See Also
2192 --------
2193 lsst.ip.isr.isrFunctions.makeThresholdMask
2194
2195 Notes
2196 -----
2197 Suspect pixels are pixels whose value is greater than
2198 amp.getSuspectLevel(). This is intended to indicate pixels that may be
2199 affected by unknown systematics; for example if non-linearity
2200 corrections above a certain level are unstable then that would be a
2201 useful value for suspectLevel. A value of `nan` indicates that no such
2202 level exists and no pixels are to be masked as suspicious.
2203 """
2204 suspectLevel = amp.getSuspectLevel()
2205 if math.isnan(suspectLevel):
2206 return
2207
2208 maskedImage = exposure.getMaskedImage()
2209 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2210 isrFunctions.makeThresholdMask(
2211 maskedImage=dataView,
2212 threshold=suspectLevel,
2213 growFootprints=0,
2214 maskName=self.config.suspectMaskName,
2215 )
2216
2217 def maskDefect(self, exposure, defectBaseList):
2218 """Mask defects using mask plane "BAD", in place.
2219
2220 Parameters
2221 ----------
2222 exposure : `lsst.afw.image.Exposure`
2223 Exposure to process.
2224 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2226 List of defects to mask.
2227
2228 Notes
2229 -----
2230 Call this after CCD assembly, since defects may cross amplifier
2231 boundaries.
2232 """
2233 maskedImage = exposure.getMaskedImage()
2234 if not isinstance(defectBaseList, Defects):
2235 # Promotes DefectBase to Defect
2236 defectList = Defects(defectBaseList)
2237 else:
2238 defectList = defectBaseList
2239 defectList.maskPixels(maskedImage, maskName="BAD")
2240
2241 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2242 """Mask edge pixels with applicable mask plane.
2243
2244 Parameters
2245 ----------
2246 exposure : `lsst.afw.image.Exposure`
2247 Exposure to process.
2248 numEdgePixels : `int`, optional
2249 Number of edge pixels to mask.
2250 maskPlane : `str`, optional
2251 Mask plane name to use.
2252 level : `str`, optional
2253 Level at which to mask edges.
2254 """
2255 maskedImage = exposure.getMaskedImage()
2256 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2257
2258 if numEdgePixels > 0:
2259 if level == 'DETECTOR':
2260 boxes = [maskedImage.getBBox()]
2261 elif level == 'AMP':
2262 boxes = [amp.getBBox() for amp in exposure.getDetector()]
2263
2264 for box in boxes:
2265 # This makes a bbox numEdgeSuspect pixels smaller than the
2266 # image on each side
2267 subImage = maskedImage[box]
2268 box.grow(-numEdgePixels)
2269 # Mask pixels outside box
2270 SourceDetectionTask.setEdgeBits(
2271 subImage,
2272 box,
2273 maskBitMask)
2274
2275 def maskAndInterpolateDefects(self, exposure, defectBaseList):
2276 """Mask and interpolate defects using mask plane "BAD", in place.
2277
2278 Parameters
2279 ----------
2280 exposure : `lsst.afw.image.Exposure`
2281 Exposure to process.
2282 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2284 List of defects to mask and interpolate.
2285
2286 See Also
2287 --------
2288 lsst.ip.isr.isrTask.maskDefect
2289 """
2290 self.maskDefect(exposure, defectBaseList)
2291 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2292 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2293 isrFunctions.interpolateFromMask(
2294 maskedImage=exposure.getMaskedImage(),
2295 fwhm=self.config.fwhm,
2296 growSaturatedFootprints=0,
2297 maskNameList=["BAD"],
2298 )
2299
2300 def maskNan(self, exposure):
2301 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2302
2303 Parameters
2304 ----------
2305 exposure : `lsst.afw.image.Exposure`
2306 Exposure to process.
2307
2308 Notes
2309 -----
2310 We mask over all non-finite values (NaN, inf), including those
2311 that are masked with other bits (because those may or may not be
2312 interpolated over later, and we want to remove all NaN/infs).
2313 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2314 preserve the historical name.
2315 """
2316 maskedImage = exposure.getMaskedImage()
2317
2318 # Find and mask NaNs
2319 maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2320 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2321 numNans = maskNans(maskedImage, maskVal)
2322 self.metadata["NUMNANS"] = numNans
2323 if numNans > 0:
2324 self.log.warning("There were %d unmasked NaNs.", numNans)
2325
2326 def maskAndInterpolateNan(self, exposure):
2327 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2328 in place.
2329
2330 Parameters
2331 ----------
2332 exposure : `lsst.afw.image.Exposure`
2333 Exposure to process.
2334
2335 See Also
2336 --------
2337 lsst.ip.isr.isrTask.maskNan
2338 """
2339 self.maskNan(exposure)
2340 isrFunctions.interpolateFromMask(
2341 maskedImage=exposure.getMaskedImage(),
2342 fwhm=self.config.fwhm,
2343 growSaturatedFootprints=0,
2344 maskNameList=["UNMASKEDNAN"],
2345 )
2346
2347 def measureBackground(self, exposure, IsrQaConfig=None):
2348 """Measure the image background in subgrids, for quality control.
2349
2350 Parameters
2351 ----------
2352 exposure : `lsst.afw.image.Exposure`
2353 Exposure to process.
2354 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2355 Configuration object containing parameters on which background
2356 statistics and subgrids to use.
2357 """
2358 if IsrQaConfig is not None:
2359 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2360 IsrQaConfig.flatness.nIter)
2361 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2362 statsControl.setAndMask(maskVal)
2363 maskedImage = exposure.getMaskedImage()
2364 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2365 skyLevel = stats.getValue(afwMath.MEDIAN)
2366 skySigma = stats.getValue(afwMath.STDEVCLIP)
2367 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2368 metadata = exposure.getMetadata()
2369 metadata["SKYLEVEL"] = skyLevel
2370 metadata["SKYSIGMA"] = skySigma
2371
2372 # calcluating flatlevel over the subgrids
2373 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2374 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2375 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2376 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2377 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2378 skyLevels = numpy.zeros((nX, nY))
2379
2380 for j in range(nY):
2381 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2382 for i in range(nX):
2383 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2384
2385 xLLC = xc - meshXHalf
2386 yLLC = yc - meshYHalf
2387 xURC = xc + meshXHalf - 1
2388 yURC = yc + meshYHalf - 1
2389
2390 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2391 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2392
2393 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2394
2395 good = numpy.where(numpy.isfinite(skyLevels))
2396 skyMedian = numpy.median(skyLevels[good])
2397 flatness = (skyLevels[good] - skyMedian) / skyMedian
2398 flatness_rms = numpy.std(flatness)
2399 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2400
2401 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2402 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2403 nX, nY, flatness_pp, flatness_rms)
2404
2405 metadata["FLATNESS_PP"] = float(flatness_pp)
2406 metadata["FLATNESS_RMS"] = float(flatness_rms)
2407 metadata["FLATNESS_NGRIDS"] = '%dx%d' % (nX, nY)
2408 metadata["FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX
2409 metadata["FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY
2410
2411 def roughZeroPoint(self, exposure):
2412 """Set an approximate magnitude zero point for the exposure.
2413
2414 Parameters
2415 ----------
2416 exposure : `lsst.afw.image.Exposure`
2417 Exposure to process.
2418 """
2419 filterLabel = exposure.getFilter()
2420 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2421
2422 if physicalFilter in self.config.fluxMag0T1:
2423 fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2424 else:
2425 self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter)
2426 fluxMag0 = self.config.defaultFluxMag0T1
2427
2428 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2429 if not expTime > 0: # handle NaN as well as <= 0
2430 self.log.warning("Non-positive exposure time; skipping rough zero point.")
2431 return
2432
2433 self.log.info("Setting rough magnitude zero point for filter %s: %f",
2434 physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2435 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2436
2437 @contextmanager
2438 def flatContext(self, exp, flat, dark=None):
2439 """Context manager that applies and removes flats and darks,
2440 if the task is configured to apply them.
2441
2442 Parameters
2443 ----------
2445 Exposure to process.
2447 Flat exposure the same size as ``exp``.
2448 dark : `lsst.afw.image.Exposure`, optional
2449 Dark exposure the same size as ``exp``.
2450
2451 Yields
2452 ------
2454 The flat and dark corrected exposure.
2455 """
2456 if self.config.doDark and dark is not None:
2457 self.darkCorrection(exp, dark)
2458 if self.config.doFlat:
2459 self.flatCorrection(exp, flat)
2460 try:
2461 yield exp
2462 finally:
2463 if self.config.doFlat:
2464 self.flatCorrection(exp, flat, invert=True)
2465 if self.config.doDark and dark is not None:
2466 self.darkCorrection(exp, dark, invert=True)
2467
2468 def debugView(self, exposure, stepname):
2469 """Utility function to examine ISR exposure at different stages.
2470
2471 Parameters
2472 ----------
2473 exposure : `lsst.afw.image.Exposure`
2474 Exposure to view.
2475 stepname : `str`
2476 State of processing to view.
2477 """
2478 frame = getDebugFrame(self._display, stepname)
2479 if frame:
2480 display = getDisplay(frame)
2481 display.scale('asinh', 'zscale')
2482 display.mtv(exposure)
2483 prompt = "Press Enter to continue [c]... "
2484 while True:
2485 ans = input(prompt).lower()
2486 if ans in ("", "c",):
2487 break
2488
2489
2490class FakeAmp(object):
2491 """A Detector-like object that supports returning gain and saturation level
2492
2493 This is used when the input exposure does not have a detector.
2494
2495 Parameters
2496 ----------
2497 exposure : `lsst.afw.image.Exposure`
2498 Exposure to generate a fake amplifier for.
2499 config : `lsst.ip.isr.isrTaskConfig`
2500 Configuration to apply to the fake amplifier.
2501 """
2502
2503 def __init__(self, exposure, config):
2504 self._bbox = exposure.getBBox(afwImage.LOCAL)
2506 self._gain = config.gain
2507 self._readNoise = config.readNoise
2508 self._saturation = config.saturation
2509
2510 def getBBox(self):
2511 return self._bbox
2512
2513 def getRawBBox(self):
2514 return self._bbox
2515
2517 return self._RawHorizontalOverscanBBox
2518
2519 def getGain(self):
2520 return self._gain
2521
2522 def getReadNoise(self):
2523 return self._readNoise
2524
2525 def getSaturation(self):
2526 return self._saturation
2527
2529 return float("NaN")
def getRawHorizontalOverscanBBox(self)
Definition: isrTask.py:2516
def __init__(self, exposure, config)
Definition: isrTask.py:2503
def __init__(self, *config=None)
Definition: isrTask.py:283
def flatCorrection(self, exposure, flatExposure, invert=False)
Definition: isrTask.py:2107
def maskAndInterpolateNan(self, exposure)
Definition: isrTask.py:2326
def saturationInterpolation(self, exposure)
Definition: isrTask.py:2156
def maskNan(self, exposure)
Definition: isrTask.py:2300
def maskAmplifier(self, ccdExposure, amp, defects)
Definition: isrTask.py:1734
def debugView(self, exposure, stepname)
Definition: isrTask.py:2468
def ensureExposure(self, inputExp, camera=None, detectorNum=None)
Definition: isrTask.py:1645
def maskNegativeVariance(self, exposure)
Definition: isrTask.py:2029
def saturationDetection(self, exposure, amp)
Definition: isrTask.py:2132
def maskDefect(self, exposure, defectBaseList)
Definition: isrTask.py:2217
def __init__(self, **kwargs)
Definition: isrTask.py:1021
def runQuantum(self, butlerQC, inputRefs, outputRefs)
Definition: isrTask.py:1034
def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR')
Definition: isrTask.py:2241
def overscanCorrection(self, ccdExposure, amp)
Definition: isrTask.py:1809
def run(self, ccdExposure, *camera=None, bias=None, linearizer=None, crosstalk=None, crosstalkSources=None, dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None, fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None, sensorTransmission=None, atmosphereTransmission=None, detectorNum=None, strayLightData=None, illumMaskedImage=None, deferredCharge=None)
Definition: isrTask.py:1139
def measureBackground(self, exposure, IsrQaConfig=None)
Definition: isrTask.py:2347
def roughZeroPoint(self, exposure)
Definition: isrTask.py:2411
def maskAndInterpolateDefects(self, exposure, defectBaseList)
Definition: isrTask.py:2275
def doLinearize(self, detector)
Definition: isrTask.py:2088
def flatContext(self, exp, flat, dark=None)
Definition: isrTask.py:2438
def convertIntToFloat(self, exposure)
Definition: isrTask.py:1697
def suspectDetection(self, exposure, amp)
Definition: isrTask.py:2181
def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None)
Definition: isrTask.py:1954
def darkCorrection(self, exposure, darkExposure, invert=False)
Definition: isrTask.py:2045
def checkFilter(exposure, filterList, log)
def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections)
Definition: isrTask.py:62
size_t maskNans(afw::image::MaskedImage< PixelT > const &mi, afw::image::MaskPixel maskVal, afw::image::MaskPixel allow=0)
Mask NANs in an image.
Definition: Isr.cc:35