Coverage for python/lsst/ip/isr/isrTask.py: 17%
944 statements
« prev ^ index » next coverage.py v6.4.2, created at 2022-07-27 02:18 -0700
« prev ^ index » next coverage.py v6.4.2, created at 2022-07-27 02:18 -0700
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import math
23import numpy
25import lsst.geom
26import lsst.afw.image as afwImage
27import lsst.afw.math as afwMath
28import lsst.pex.config as pexConfig
29import lsst.pipe.base as pipeBase
30import lsst.pipe.base.connectionTypes as cT
32from contextlib import contextmanager
33from lsstDebug import getDebugFrame
35from lsst.afw.cameraGeom import NullLinearityType, ReadoutCorner
36from lsst.afw.display import getDisplay
37from lsst.daf.persistence import ButlerDataRef
38from lsst.daf.persistence.butler import NoResults
39from lsst.meas.algorithms.detection import SourceDetectionTask
40from lsst.utils.timer import timeMethod
42from . import isrFunctions
43from . import isrQa
44from . import linearize
45from .defects import Defects
47from .assembleCcdTask import AssembleCcdTask
48from .crosstalk import CrosstalkTask, CrosstalkCalib
49from .fringe import FringeTask
50from .isr import maskNans
51from .masking import MaskingTask
52from .overscan import OverscanCorrectionTask
53from .straylight import StrayLightTask
54from .vignette import VignetteTask
55from .ampOffset import AmpOffsetTask
56from .deferredCharge import DeferredChargeTask
57from .isrStatistics import IsrStatisticsTask
58from lsst.daf.butler import DimensionGraph
61__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
64def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections):
65 """Lookup function to identify crosstalkSource entries.
67 This should return an empty list under most circumstances. Only
68 when inter-chip crosstalk has been identified should this be
69 populated.
71 Parameters
72 ----------
73 datasetType : `str`
74 Dataset to lookup.
75 registry : `lsst.daf.butler.Registry`
76 Butler registry to query.
77 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
78 Data id to transform to identify crosstalkSources. The
79 ``detector`` entry will be stripped.
80 collections : `lsst.daf.butler.CollectionSearch`
81 Collections to search through.
83 Returns
84 -------
85 results : `list` [`lsst.daf.butler.DatasetRef`]
86 List of datasets that match the query that will be used as
87 crosstalkSources.
88 """
89 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"]))
90 results = set(registry.queryDatasets(datasetType, collections=collections, dataId=newDataId,
91 findFirst=True))
92 # In some contexts, calling `.expanded()` to expand all data IDs in the
93 # query results can be a lot faster because it vectorizes lookups. But in
94 # this case, expandDataId shouldn't need to hit the database at all in the
95 # steady state, because only the detector record is unknown and those are
96 # cached in the registry.
97 return [ref.expanded(registry.expandDataId(ref.dataId, records=newDataId.records)) for ref in results]
100class IsrTaskConnections(pipeBase.PipelineTaskConnections,
101 dimensions={"instrument", "exposure", "detector"},
102 defaultTemplates={}):
103 ccdExposure = cT.Input(
104 name="raw",
105 doc="Input exposure to process.",
106 storageClass="Exposure",
107 dimensions=["instrument", "exposure", "detector"],
108 )
109 camera = cT.PrerequisiteInput(
110 name="camera",
111 storageClass="Camera",
112 doc="Input camera to construct complete exposures.",
113 dimensions=["instrument"],
114 isCalibration=True,
115 )
117 crosstalk = cT.PrerequisiteInput(
118 name="crosstalk",
119 doc="Input crosstalk object",
120 storageClass="CrosstalkCalib",
121 dimensions=["instrument", "detector"],
122 isCalibration=True,
123 minimum=0, # can fall back to cameraGeom
124 )
125 crosstalkSources = cT.PrerequisiteInput(
126 name="isrOverscanCorrected",
127 doc="Overscan corrected input images.",
128 storageClass="Exposure",
129 dimensions=["instrument", "exposure", "detector"],
130 deferLoad=True,
131 multiple=True,
132 lookupFunction=crosstalkSourceLookup,
133 minimum=0, # not needed for all instruments, no config to control this
134 )
135 bias = cT.PrerequisiteInput(
136 name="bias",
137 doc="Input bias calibration.",
138 storageClass="ExposureF",
139 dimensions=["instrument", "detector"],
140 isCalibration=True,
141 )
142 dark = cT.PrerequisiteInput(
143 name='dark',
144 doc="Input dark calibration.",
145 storageClass="ExposureF",
146 dimensions=["instrument", "detector"],
147 isCalibration=True,
148 )
149 flat = cT.PrerequisiteInput(
150 name="flat",
151 doc="Input flat calibration.",
152 storageClass="ExposureF",
153 dimensions=["instrument", "physical_filter", "detector"],
154 isCalibration=True,
155 )
156 ptc = cT.PrerequisiteInput(
157 name="ptc",
158 doc="Input Photon Transfer Curve dataset",
159 storageClass="PhotonTransferCurveDataset",
160 dimensions=["instrument", "detector"],
161 isCalibration=True,
162 )
163 fringes = cT.PrerequisiteInput(
164 name="fringe",
165 doc="Input fringe calibration.",
166 storageClass="ExposureF",
167 dimensions=["instrument", "physical_filter", "detector"],
168 isCalibration=True,
169 minimum=0, # only needed for some bands, even when enabled
170 )
171 strayLightData = cT.PrerequisiteInput(
172 name='yBackground',
173 doc="Input stray light calibration.",
174 storageClass="StrayLightData",
175 dimensions=["instrument", "physical_filter", "detector"],
176 deferLoad=True,
177 isCalibration=True,
178 minimum=0, # only needed for some bands, even when enabled
179 )
180 bfKernel = cT.PrerequisiteInput(
181 name='bfKernel',
182 doc="Input brighter-fatter kernel.",
183 storageClass="NumpyArray",
184 dimensions=["instrument"],
185 isCalibration=True,
186 minimum=0, # can use either bfKernel or newBFKernel
187 )
188 newBFKernel = cT.PrerequisiteInput(
189 name='brighterFatterKernel',
190 doc="Newer complete kernel + gain solutions.",
191 storageClass="BrighterFatterKernel",
192 dimensions=["instrument", "detector"],
193 isCalibration=True,
194 minimum=0, # can use either bfKernel or newBFKernel
195 )
196 defects = cT.PrerequisiteInput(
197 name='defects',
198 doc="Input defect tables.",
199 storageClass="Defects",
200 dimensions=["instrument", "detector"],
201 isCalibration=True,
202 )
203 linearizer = cT.PrerequisiteInput(
204 name='linearizer',
205 storageClass="Linearizer",
206 doc="Linearity correction calibration.",
207 dimensions=["instrument", "detector"],
208 isCalibration=True,
209 minimum=0, # can fall back to cameraGeom
210 )
211 opticsTransmission = cT.PrerequisiteInput(
212 name="transmission_optics",
213 storageClass="TransmissionCurve",
214 doc="Transmission curve due to the optics.",
215 dimensions=["instrument"],
216 isCalibration=True,
217 )
218 filterTransmission = cT.PrerequisiteInput(
219 name="transmission_filter",
220 storageClass="TransmissionCurve",
221 doc="Transmission curve due to the filter.",
222 dimensions=["instrument", "physical_filter"],
223 isCalibration=True,
224 )
225 sensorTransmission = cT.PrerequisiteInput(
226 name="transmission_sensor",
227 storageClass="TransmissionCurve",
228 doc="Transmission curve due to the sensor.",
229 dimensions=["instrument", "detector"],
230 isCalibration=True,
231 )
232 atmosphereTransmission = cT.PrerequisiteInput(
233 name="transmission_atmosphere",
234 storageClass="TransmissionCurve",
235 doc="Transmission curve due to the atmosphere.",
236 dimensions=["instrument"],
237 isCalibration=True,
238 )
239 illumMaskedImage = cT.PrerequisiteInput(
240 name="illum",
241 doc="Input illumination correction.",
242 storageClass="MaskedImageF",
243 dimensions=["instrument", "physical_filter", "detector"],
244 isCalibration=True,
245 )
246 deferredChargeCalib = cT.PrerequisiteInput(
247 name="deferredCharge",
248 doc="Deferred charge/CTI correction dataset.",
249 storageClass="IsrCalib",
250 dimensions=["instrument", "detector"],
251 isCalibration=True,
252 )
254 outputExposure = cT.Output(
255 name='postISRCCD',
256 doc="Output ISR processed exposure.",
257 storageClass="Exposure",
258 dimensions=["instrument", "exposure", "detector"],
259 )
260 preInterpExposure = cT.Output(
261 name='preInterpISRCCD',
262 doc="Output ISR processed exposure, with pixels left uninterpolated.",
263 storageClass="ExposureF",
264 dimensions=["instrument", "exposure", "detector"],
265 )
266 outputOssThumbnail = cT.Output(
267 name="OssThumb",
268 doc="Output Overscan-subtracted thumbnail image.",
269 storageClass="Thumbnail",
270 dimensions=["instrument", "exposure", "detector"],
271 )
272 outputFlattenedThumbnail = cT.Output(
273 name="FlattenedThumb",
274 doc="Output flat-corrected thumbnail image.",
275 storageClass="Thumbnail",
276 dimensions=["instrument", "exposure", "detector"],
277 )
278 outputStatistics = cT.Output(
279 name="isrStatistics",
280 doc="Output of additional statistics table.",
281 storageClass="StructuredDataDict",
282 dimensions=["instrument", "exposure", "detector"],
283 )
285 def __init__(self, *, config=None):
286 super().__init__(config=config)
288 if config.doBias is not True:
289 self.prerequisiteInputs.remove("bias")
290 if config.doLinearize is not True:
291 self.prerequisiteInputs.remove("linearizer")
292 if config.doCrosstalk is not True:
293 self.prerequisiteInputs.remove("crosstalkSources")
294 self.prerequisiteInputs.remove("crosstalk")
295 if config.doBrighterFatter is not True:
296 self.prerequisiteInputs.remove("bfKernel")
297 self.prerequisiteInputs.remove("newBFKernel")
298 if config.doDefect is not True:
299 self.prerequisiteInputs.remove("defects")
300 if config.doDark is not True:
301 self.prerequisiteInputs.remove("dark")
302 if config.doFlat is not True:
303 self.prerequisiteInputs.remove("flat")
304 if config.doFringe is not True:
305 self.prerequisiteInputs.remove("fringes")
306 if config.doStrayLight is not True:
307 self.prerequisiteInputs.remove("strayLightData")
308 if config.usePtcGains is not True and config.usePtcReadNoise is not True:
309 self.prerequisiteInputs.remove("ptc")
310 if config.doAttachTransmissionCurve is not True:
311 self.prerequisiteInputs.remove("opticsTransmission")
312 self.prerequisiteInputs.remove("filterTransmission")
313 self.prerequisiteInputs.remove("sensorTransmission")
314 self.prerequisiteInputs.remove("atmosphereTransmission")
315 else:
316 if config.doUseOpticsTransmission is not True:
317 self.prerequisiteInputs.remove("opticsTransmission")
318 if config.doUseFilterTransmission is not True:
319 self.prerequisiteInputs.remove("filterTransmission")
320 if config.doUseSensorTransmission is not True:
321 self.prerequisiteInputs.remove("sensorTransmission")
322 if config.doUseAtmosphereTransmission is not True:
323 self.prerequisiteInputs.remove("atmosphereTransmission")
324 if config.doIlluminationCorrection is not True:
325 self.prerequisiteInputs.remove("illumMaskedImage")
326 if config.doDeferredCharge is not True:
327 self.prerequisiteInputs.remove("deferredChargeCalib")
329 if config.doWrite is not True:
330 self.outputs.remove("outputExposure")
331 self.outputs.remove("preInterpExposure")
332 self.outputs.remove("outputFlattenedThumbnail")
333 self.outputs.remove("outputOssThumbnail")
334 self.outputs.remove("outputStatistics")
336 if config.doSaveInterpPixels is not True:
337 self.outputs.remove("preInterpExposure")
338 if config.qa.doThumbnailOss is not True:
339 self.outputs.remove("outputOssThumbnail")
340 if config.qa.doThumbnailFlattened is not True:
341 self.outputs.remove("outputFlattenedThumbnail")
342 if config.doCalculateStatistics is not True:
343 self.outputs.remove("outputStatistics")
346class IsrTaskConfig(pipeBase.PipelineTaskConfig,
347 pipelineConnections=IsrTaskConnections):
348 """Configuration parameters for IsrTask.
350 Items are grouped in the order in which they are executed by the task.
351 """
352 datasetType = pexConfig.Field(
353 dtype=str,
354 doc="Dataset type for input data; users will typically leave this alone, "
355 "but camera-specific ISR tasks will override it",
356 default="raw",
357 )
359 fallbackFilterName = pexConfig.Field(
360 dtype=str,
361 doc="Fallback default filter name for calibrations.",
362 optional=True
363 )
364 useFallbackDate = pexConfig.Field(
365 dtype=bool,
366 doc="Pass observation date when using fallback filter.",
367 default=False,
368 )
369 expectWcs = pexConfig.Field(
370 dtype=bool,
371 default=True,
372 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
373 )
374 fwhm = pexConfig.Field(
375 dtype=float,
376 doc="FWHM of PSF in arcseconds.",
377 default=1.0,
378 )
379 qa = pexConfig.ConfigField(
380 dtype=isrQa.IsrQaConfig,
381 doc="QA related configuration options.",
382 )
384 # Image conversion configuration
385 doConvertIntToFloat = pexConfig.Field(
386 dtype=bool,
387 doc="Convert integer raw images to floating point values?",
388 default=True,
389 )
391 # Saturated pixel handling.
392 doSaturation = pexConfig.Field(
393 dtype=bool,
394 doc="Mask saturated pixels? NB: this is totally independent of the"
395 " interpolation option - this is ONLY setting the bits in the mask."
396 " To have them interpolated make sure doSaturationInterpolation=True",
397 default=True,
398 )
399 saturatedMaskName = pexConfig.Field(
400 dtype=str,
401 doc="Name of mask plane to use in saturation detection and interpolation",
402 default="SAT",
403 )
404 saturation = pexConfig.Field(
405 dtype=float,
406 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
407 default=float("NaN"),
408 )
409 growSaturationFootprintSize = pexConfig.Field(
410 dtype=int,
411 doc="Number of pixels by which to grow the saturation footprints",
412 default=1,
413 )
415 # Suspect pixel handling.
416 doSuspect = pexConfig.Field(
417 dtype=bool,
418 doc="Mask suspect pixels?",
419 default=False,
420 )
421 suspectMaskName = pexConfig.Field(
422 dtype=str,
423 doc="Name of mask plane to use for suspect pixels",
424 default="SUSPECT",
425 )
426 numEdgeSuspect = pexConfig.Field(
427 dtype=int,
428 doc="Number of edge pixels to be flagged as untrustworthy.",
429 default=0,
430 )
431 edgeMaskLevel = pexConfig.ChoiceField(
432 dtype=str,
433 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
434 default="DETECTOR",
435 allowed={
436 'DETECTOR': 'Mask only the edges of the full detector.',
437 'AMP': 'Mask edges of each amplifier.',
438 },
439 )
441 # Initial masking options.
442 doSetBadRegions = pexConfig.Field(
443 dtype=bool,
444 doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
445 default=True,
446 )
447 badStatistic = pexConfig.ChoiceField(
448 dtype=str,
449 doc="How to estimate the average value for BAD regions.",
450 default='MEANCLIP',
451 allowed={
452 "MEANCLIP": "Correct using the (clipped) mean of good data",
453 "MEDIAN": "Correct using the median of the good data",
454 },
455 )
457 # Overscan subtraction configuration.
458 doOverscan = pexConfig.Field(
459 dtype=bool,
460 doc="Do overscan subtraction?",
461 default=True,
462 )
463 overscan = pexConfig.ConfigurableField(
464 target=OverscanCorrectionTask,
465 doc="Overscan subtraction task for image segments.",
466 )
467 overscanFitType = pexConfig.ChoiceField(
468 dtype=str,
469 doc="The method for fitting the overscan bias level.",
470 default='MEDIAN',
471 allowed={
472 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
473 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
474 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
475 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
476 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
477 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
478 "MEAN": "Correct using the mean of the overscan region",
479 "MEANCLIP": "Correct using a clipped mean of the overscan region",
480 "MEDIAN": "Correct using the median of the overscan region",
481 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
482 },
483 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
484 " This option will no longer be used, and will be removed after v20.")
485 )
486 overscanOrder = pexConfig.Field(
487 dtype=int,
488 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, "
489 "or number of spline knots if overscan fit type is a spline."),
490 default=1,
491 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
492 " This option will no longer be used, and will be removed after v20.")
493 )
494 overscanNumSigmaClip = pexConfig.Field(
495 dtype=float,
496 doc="Rejection threshold (sigma) for collapsing overscan before fit",
497 default=3.0,
498 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
499 " This option will no longer be used, and will be removed after v20.")
500 )
501 overscanIsInt = pexConfig.Field(
502 dtype=bool,
503 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
504 " and overscan.FitType=MEDIAN_PER_ROW.",
505 default=True,
506 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
507 " This option will no longer be used, and will be removed after v20.")
508 )
509 # These options do not get deprecated, as they define how we slice up the
510 # image data.
511 overscanNumLeadingColumnsToSkip = pexConfig.Field(
512 dtype=int,
513 doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
514 default=0,
515 )
516 overscanNumTrailingColumnsToSkip = pexConfig.Field(
517 dtype=int,
518 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
519 default=0,
520 )
521 overscanMaxDev = pexConfig.Field( 521 ↛ exitline 521 didn't jump to the function exit
522 dtype=float,
523 doc="Maximum deviation from the median for overscan",
524 default=1000.0, check=lambda x: x > 0
525 )
526 overscanBiasJump = pexConfig.Field(
527 dtype=bool,
528 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
529 default=False,
530 )
531 overscanBiasJumpKeyword = pexConfig.Field(
532 dtype=str,
533 doc="Header keyword containing information about devices.",
534 default="NO_SUCH_KEY",
535 )
536 overscanBiasJumpDevices = pexConfig.ListField(
537 dtype=str,
538 doc="List of devices that need piecewise overscan correction.",
539 default=(),
540 )
541 overscanBiasJumpLocation = pexConfig.Field(
542 dtype=int,
543 doc="Location of bias jump along y-axis.",
544 default=0,
545 )
547 # Amplifier to CCD assembly configuration
548 doAssembleCcd = pexConfig.Field(
549 dtype=bool,
550 default=True,
551 doc="Assemble amp-level exposures into a ccd-level exposure?"
552 )
553 assembleCcd = pexConfig.ConfigurableField(
554 target=AssembleCcdTask,
555 doc="CCD assembly task",
556 )
558 # General calibration configuration.
559 doAssembleIsrExposures = pexConfig.Field(
560 dtype=bool,
561 default=False,
562 doc="Assemble amp-level calibration exposures into ccd-level exposure?"
563 )
564 doTrimToMatchCalib = pexConfig.Field(
565 dtype=bool,
566 default=False,
567 doc="Trim raw data to match calibration bounding boxes?"
568 )
570 # Bias subtraction.
571 doBias = pexConfig.Field(
572 dtype=bool,
573 doc="Apply bias frame correction?",
574 default=True,
575 )
576 biasDataProductName = pexConfig.Field(
577 dtype=str,
578 doc="Name of the bias data product",
579 default="bias",
580 )
581 doBiasBeforeOverscan = pexConfig.Field(
582 dtype=bool,
583 doc="Reverse order of overscan and bias correction.",
584 default=False
585 )
587 # Deferred charge correction.
588 doDeferredCharge = pexConfig.Field(
589 dtype=bool,
590 doc="Apply deferred charge correction?",
591 default=False,
592 )
593 deferredChargeCorrection = pexConfig.ConfigurableField(
594 target=DeferredChargeTask,
595 doc="Deferred charge correction task.",
596 )
598 # Variance construction
599 doVariance = pexConfig.Field(
600 dtype=bool,
601 doc="Calculate variance?",
602 default=True
603 )
604 gain = pexConfig.Field(
605 dtype=float,
606 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
607 default=float("NaN"),
608 )
609 readNoise = pexConfig.Field(
610 dtype=float,
611 doc="The read noise to use if no Detector is present in the Exposure",
612 default=0.0,
613 )
614 doEmpiricalReadNoise = pexConfig.Field(
615 dtype=bool,
616 default=False,
617 doc="Calculate empirical read noise instead of value from AmpInfo data?"
618 )
619 usePtcReadNoise = pexConfig.Field(
620 dtype=bool,
621 default=False,
622 doc="Use readnoise values from the Photon Transfer Curve?"
623 )
624 maskNegativeVariance = pexConfig.Field(
625 dtype=bool,
626 default=True,
627 doc="Mask pixels that claim a negative variance? This likely indicates a failure "
628 "in the measurement of the overscan at an edge due to the data falling off faster "
629 "than the overscan model can account for it."
630 )
631 negativeVarianceMaskName = pexConfig.Field(
632 dtype=str,
633 default="BAD",
634 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
635 )
636 # Linearization.
637 doLinearize = pexConfig.Field(
638 dtype=bool,
639 doc="Correct for nonlinearity of the detector's response?",
640 default=True,
641 )
643 # Crosstalk.
644 doCrosstalk = pexConfig.Field(
645 dtype=bool,
646 doc="Apply intra-CCD crosstalk correction?",
647 default=False,
648 )
649 doCrosstalkBeforeAssemble = pexConfig.Field(
650 dtype=bool,
651 doc="Apply crosstalk correction before CCD assembly, and before trimming?",
652 default=False,
653 )
654 crosstalk = pexConfig.ConfigurableField(
655 target=CrosstalkTask,
656 doc="Intra-CCD crosstalk correction",
657 )
659 # Masking options.
660 doDefect = pexConfig.Field(
661 dtype=bool,
662 doc="Apply correction for CCD defects, e.g. hot pixels?",
663 default=True,
664 )
665 doNanMasking = pexConfig.Field(
666 dtype=bool,
667 doc="Mask non-finite (NAN, inf) pixels?",
668 default=True,
669 )
670 doWidenSaturationTrails = pexConfig.Field(
671 dtype=bool,
672 doc="Widen bleed trails based on their width?",
673 default=True
674 )
676 # Brighter-Fatter correction.
677 doBrighterFatter = pexConfig.Field(
678 dtype=bool,
679 default=False,
680 doc="Apply the brighter-fatter correction?"
681 )
682 brighterFatterLevel = pexConfig.ChoiceField(
683 dtype=str,
684 default="DETECTOR",
685 doc="The level at which to correct for brighter-fatter.",
686 allowed={
687 "AMP": "Every amplifier treated separately.",
688 "DETECTOR": "One kernel per detector",
689 }
690 )
691 brighterFatterMaxIter = pexConfig.Field(
692 dtype=int,
693 default=10,
694 doc="Maximum number of iterations for the brighter-fatter correction"
695 )
696 brighterFatterThreshold = pexConfig.Field(
697 dtype=float,
698 default=1000,
699 doc="Threshold used to stop iterating the brighter-fatter correction. It is the "
700 "absolute value of the difference between the current corrected image and the one "
701 "from the previous iteration summed over all the pixels."
702 )
703 brighterFatterApplyGain = pexConfig.Field(
704 dtype=bool,
705 default=True,
706 doc="Should the gain be applied when applying the brighter-fatter correction?"
707 )
708 brighterFatterMaskListToInterpolate = pexConfig.ListField(
709 dtype=str,
710 doc="List of mask planes that should be interpolated over when applying the brighter-fatter "
711 "correction.",
712 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"],
713 )
714 brighterFatterMaskGrowSize = pexConfig.Field(
715 dtype=int,
716 default=0,
717 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
718 "when brighter-fatter correction is applied."
719 )
721 # Dark subtraction.
722 doDark = pexConfig.Field(
723 dtype=bool,
724 doc="Apply dark frame correction?",
725 default=True,
726 )
727 darkDataProductName = pexConfig.Field(
728 dtype=str,
729 doc="Name of the dark data product",
730 default="dark",
731 )
733 # Camera-specific stray light removal.
734 doStrayLight = pexConfig.Field(
735 dtype=bool,
736 doc="Subtract stray light in the y-band (due to encoder LEDs)?",
737 default=False,
738 )
739 strayLight = pexConfig.ConfigurableField(
740 target=StrayLightTask,
741 doc="y-band stray light correction"
742 )
744 # Flat correction.
745 doFlat = pexConfig.Field(
746 dtype=bool,
747 doc="Apply flat field correction?",
748 default=True,
749 )
750 flatDataProductName = pexConfig.Field(
751 dtype=str,
752 doc="Name of the flat data product",
753 default="flat",
754 )
755 flatScalingType = pexConfig.ChoiceField(
756 dtype=str,
757 doc="The method for scaling the flat on the fly.",
758 default='USER',
759 allowed={
760 "USER": "Scale by flatUserScale",
761 "MEAN": "Scale by the inverse of the mean",
762 "MEDIAN": "Scale by the inverse of the median",
763 },
764 )
765 flatUserScale = pexConfig.Field(
766 dtype=float,
767 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
768 default=1.0,
769 )
770 doTweakFlat = pexConfig.Field(
771 dtype=bool,
772 doc="Tweak flats to match observed amplifier ratios?",
773 default=False
774 )
776 # Amplifier normalization based on gains instead of using flats
777 # configuration.
778 doApplyGains = pexConfig.Field(
779 dtype=bool,
780 doc="Correct the amplifiers for their gains instead of applying flat correction",
781 default=False,
782 )
783 usePtcGains = pexConfig.Field(
784 dtype=bool,
785 doc="Use the gain values from the Photon Transfer Curve?",
786 default=False,
787 )
788 normalizeGains = pexConfig.Field(
789 dtype=bool,
790 doc="Normalize all the amplifiers in each CCD to have the same median value.",
791 default=False,
792 )
794 # Fringe correction.
795 doFringe = pexConfig.Field(
796 dtype=bool,
797 doc="Apply fringe correction?",
798 default=True,
799 )
800 fringe = pexConfig.ConfigurableField(
801 target=FringeTask,
802 doc="Fringe subtraction task",
803 )
804 fringeAfterFlat = pexConfig.Field(
805 dtype=bool,
806 doc="Do fringe subtraction after flat-fielding?",
807 default=True,
808 )
810 # Amp offset correction.
811 doAmpOffset = pexConfig.Field(
812 doc="Calculate and apply amp offset corrections?",
813 dtype=bool,
814 default=False,
815 )
816 ampOffset = pexConfig.ConfigurableField(
817 doc="Amp offset correction task.",
818 target=AmpOffsetTask,
819 )
821 # Initial CCD-level background statistics options.
822 doMeasureBackground = pexConfig.Field(
823 dtype=bool,
824 doc="Measure the background level on the reduced image?",
825 default=False,
826 )
828 # Camera-specific masking configuration.
829 doCameraSpecificMasking = pexConfig.Field(
830 dtype=bool,
831 doc="Mask camera-specific bad regions?",
832 default=False,
833 )
834 masking = pexConfig.ConfigurableField(
835 target=MaskingTask,
836 doc="Masking task."
837 )
839 # Interpolation options.
840 doInterpolate = pexConfig.Field(
841 dtype=bool,
842 doc="Interpolate masked pixels?",
843 default=True,
844 )
845 doSaturationInterpolation = pexConfig.Field(
846 dtype=bool,
847 doc="Perform interpolation over pixels masked as saturated?"
848 " NB: This is independent of doSaturation; if that is False this plane"
849 " will likely be blank, resulting in a no-op here.",
850 default=True,
851 )
852 doNanInterpolation = pexConfig.Field(
853 dtype=bool,
854 doc="Perform interpolation over pixels masked as NaN?"
855 " NB: This is independent of doNanMasking; if that is False this plane"
856 " will likely be blank, resulting in a no-op here.",
857 default=True,
858 )
859 doNanInterpAfterFlat = pexConfig.Field(
860 dtype=bool,
861 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
862 "also have to interpolate them before flat-fielding."),
863 default=False,
864 )
865 maskListToInterpolate = pexConfig.ListField(
866 dtype=str,
867 doc="List of mask planes that should be interpolated.",
868 default=['SAT', 'BAD'],
869 )
870 doSaveInterpPixels = pexConfig.Field(
871 dtype=bool,
872 doc="Save a copy of the pre-interpolated pixel values?",
873 default=False,
874 )
876 # Default photometric calibration options.
877 fluxMag0T1 = pexConfig.DictField(
878 keytype=str,
879 itemtype=float,
880 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
881 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
882 ))
883 )
884 defaultFluxMag0T1 = pexConfig.Field(
885 dtype=float,
886 doc="Default value for fluxMag0T1 (for an unrecognized filter).",
887 default=pow(10.0, 0.4*28.0)
888 )
890 # Vignette correction configuration.
891 doVignette = pexConfig.Field(
892 dtype=bool,
893 doc=("Compute and attach the validPolygon defining the unvignetted region to the exposure "
894 "according to vignetting parameters?"),
895 default=False,
896 )
897 doMaskVignettePolygon = pexConfig.Field(
898 dtype=bool,
899 doc=("Add a mask bit for pixels within the vignetted region. Ignored if doVignette "
900 "is False"),
901 default=True,
902 )
903 vignetteValue = pexConfig.Field(
904 dtype=float,
905 doc="Value to replace image array pixels with in the vignetted region? Ignored if None.",
906 optional=True,
907 default=None,
908 )
909 vignette = pexConfig.ConfigurableField(
910 target=VignetteTask,
911 doc="Vignetting task.",
912 )
914 # Transmission curve configuration.
915 doAttachTransmissionCurve = pexConfig.Field(
916 dtype=bool,
917 default=False,
918 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
919 )
920 doUseOpticsTransmission = pexConfig.Field(
921 dtype=bool,
922 default=True,
923 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
924 )
925 doUseFilterTransmission = pexConfig.Field(
926 dtype=bool,
927 default=True,
928 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
929 )
930 doUseSensorTransmission = pexConfig.Field(
931 dtype=bool,
932 default=True,
933 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
934 )
935 doUseAtmosphereTransmission = pexConfig.Field(
936 dtype=bool,
937 default=True,
938 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
939 )
941 # Illumination correction.
942 doIlluminationCorrection = pexConfig.Field(
943 dtype=bool,
944 default=False,
945 doc="Perform illumination correction?"
946 )
947 illuminationCorrectionDataProductName = pexConfig.Field(
948 dtype=str,
949 doc="Name of the illumination correction data product.",
950 default="illumcor",
951 )
952 illumScale = pexConfig.Field(
953 dtype=float,
954 doc="Scale factor for the illumination correction.",
955 default=1.0,
956 )
957 illumFilters = pexConfig.ListField(
958 dtype=str,
959 default=[],
960 doc="Only perform illumination correction for these filters."
961 )
963 # Calculate additional statistics?
964 doCalculateStatistics = pexConfig.Field(
965 dtype=bool,
966 doc="Should additional ISR statistics be calculated?",
967 default=False,
968 )
969 isrStats = pexConfig.ConfigurableField(
970 target=IsrStatisticsTask,
971 doc="Task to calculate additional statistics.",
972 )
974 # Write the outputs to disk. If ISR is run as a subtask, this may not
975 # be needed.
976 doWrite = pexConfig.Field(
977 dtype=bool,
978 doc="Persist postISRCCD?",
979 default=True,
980 )
982 def validate(self):
983 super().validate()
984 if self.doFlat and self.doApplyGains:
985 raise ValueError("You may not specify both doFlat and doApplyGains")
986 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib:
987 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
988 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate:
989 self.maskListToInterpolate.append(self.saturatedMaskName)
990 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate:
991 self.maskListToInterpolate.remove(self.saturatedMaskName)
992 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
993 self.maskListToInterpolate.append("UNMASKEDNAN")
996class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
997 """Apply common instrument signature correction algorithms to a raw frame.
999 The process for correcting imaging data is very similar from
1000 camera to camera. This task provides a vanilla implementation of
1001 doing these corrections, including the ability to turn certain
1002 corrections off if they are not needed. The inputs to the primary
1003 method, `run()`, are a raw exposure to be corrected and the
1004 calibration data products. The raw input is a single chip sized
1005 mosaic of all amps including overscans and other non-science
1006 pixels. The method `runDataRef()` identifies and defines the
1007 calibration data products, and is intended for use by a
1008 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
1009 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
1010 subclassed for different camera, although the most camera specific
1011 methods have been split into subtasks that can be redirected
1012 appropriately.
1014 The __init__ method sets up the subtasks for ISR processing, using
1015 the defaults from `lsst.ip.isr`.
1017 Parameters
1018 ----------
1019 args : `list`
1020 Positional arguments passed to the Task constructor.
1021 None used at this time.
1022 kwargs : `dict`, optional
1023 Keyword arguments passed on to the Task constructor.
1024 None used at this time.
1025 """
1026 ConfigClass = IsrTaskConfig
1027 _DefaultName = "isr"
1029 def __init__(self, **kwargs):
1030 super().__init__(**kwargs)
1031 self.makeSubtask("assembleCcd")
1032 self.makeSubtask("crosstalk")
1033 self.makeSubtask("strayLight")
1034 self.makeSubtask("fringe")
1035 self.makeSubtask("masking")
1036 self.makeSubtask("overscan")
1037 self.makeSubtask("vignette")
1038 self.makeSubtask("ampOffset")
1039 self.makeSubtask("deferredChargeCorrection")
1040 self.makeSubtask("isrStats")
1042 def runQuantum(self, butlerQC, inputRefs, outputRefs):
1043 inputs = butlerQC.get(inputRefs)
1045 try:
1046 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
1047 except Exception as e:
1048 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
1049 (inputRefs, e))
1051 inputs['isGen3'] = True
1053 detector = inputs['ccdExposure'].getDetector()
1055 if self.config.doCrosstalk is True:
1056 # Crosstalk sources need to be defined by the pipeline
1057 # yaml if they exist.
1058 if 'crosstalk' in inputs and inputs['crosstalk'] is not None:
1059 if not isinstance(inputs['crosstalk'], CrosstalkCalib):
1060 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk'])
1061 else:
1062 coeffVector = (self.config.crosstalk.crosstalkValues
1063 if self.config.crosstalk.useConfigCoefficients else None)
1064 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
1065 inputs['crosstalk'] = crosstalkCalib
1066 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0:
1067 if 'crosstalkSources' not in inputs:
1068 self.log.warning("No crosstalkSources found for chip with interChip terms!")
1070 if self.doLinearize(detector) is True:
1071 if 'linearizer' in inputs:
1072 if isinstance(inputs['linearizer'], dict):
1073 linearizer = linearize.Linearizer(detector=detector, log=self.log)
1074 linearizer.fromYaml(inputs['linearizer'])
1075 self.log.warning("Dictionary linearizers will be deprecated in DM-28741.")
1076 elif isinstance(inputs['linearizer'], numpy.ndarray):
1077 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None),
1078 detector=detector,
1079 log=self.log)
1080 self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.")
1081 else:
1082 linearizer = inputs['linearizer']
1083 linearizer.log = self.log
1084 inputs['linearizer'] = linearizer
1085 else:
1086 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log)
1087 self.log.warning("Constructing linearizer from cameraGeom information.")
1089 if self.config.doDefect is True:
1090 if "defects" in inputs and inputs['defects'] is not None:
1091 # defects is loaded as a BaseCatalog with columns
1092 # x0, y0, width, height. Masking expects a list of defects
1093 # defined by their bounding box
1094 if not isinstance(inputs["defects"], Defects):
1095 inputs["defects"] = Defects.fromTable(inputs["defects"])
1097 # Load the correct style of brighter-fatter kernel, and repack
1098 # the information as a numpy array.
1099 if self.config.doBrighterFatter:
1100 brighterFatterKernel = inputs.pop('newBFKernel', None)
1101 if brighterFatterKernel is None:
1102 brighterFatterKernel = inputs.get('bfKernel', None)
1104 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1105 # This is a ISR calib kernel
1106 detName = detector.getName()
1107 level = brighterFatterKernel.level
1109 # This is expected to be a dictionary of amp-wise gains.
1110 inputs['bfGains'] = brighterFatterKernel.gain
1111 if self.config.brighterFatterLevel == 'DETECTOR':
1112 if level == 'DETECTOR':
1113 if detName in brighterFatterKernel.detKernels:
1114 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1115 else:
1116 raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1117 elif level == 'AMP':
1118 self.log.warning("Making DETECTOR level kernel from AMP based brighter "
1119 "fatter kernels.")
1120 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1121 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1122 elif self.config.brighterFatterLevel == 'AMP':
1123 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1125 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
1126 expId = inputs['ccdExposure'].info.id
1127 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
1128 expId=expId,
1129 assembler=self.assembleCcd
1130 if self.config.doAssembleIsrExposures else None)
1131 else:
1132 inputs['fringes'] = pipeBase.Struct(fringes=None)
1134 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
1135 if 'strayLightData' not in inputs:
1136 inputs['strayLightData'] = None
1138 outputs = self.run(**inputs)
1139 butlerQC.put(outputs, outputRefs)
1141 def readIsrData(self, dataRef, rawExposure):
1142 """Retrieve necessary frames for instrument signature removal.
1144 Pre-fetching all required ISR data products limits the IO
1145 required by the ISR. Any conflict between the calibration data
1146 available and that needed for ISR is also detected prior to
1147 doing processing, allowing it to fail quickly.
1149 Parameters
1150 ----------
1151 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1152 Butler reference of the detector data to be processed
1153 rawExposure : `afw.image.Exposure`
1154 The raw exposure that will later be corrected with the
1155 retrieved calibration data; should not be modified in this
1156 method.
1158 Returns
1159 -------
1160 result : `lsst.pipe.base.Struct`
1161 Result struct with components (which may be `None`):
1162 - ``bias``: bias calibration frame (`afw.image.Exposure`)
1163 - ``linearizer``: functor for linearization
1164 (`ip.isr.linearize.LinearizeBase`)
1165 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1166 - ``dark``: dark calibration frame (`afw.image.Exposure`)
1167 - ``flat``: flat calibration frame (`afw.image.Exposure`)
1168 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1169 - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1170 - ``fringes``: `lsst.pipe.base.Struct` with components:
1171 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1172 - ``seed``: random seed derived from the ccdExposureId for random
1173 number generator (`uint32`).
1174 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1175 A ``TransmissionCurve`` that represents the throughput of the
1176 optics, to be evaluated in focal-plane coordinates.
1177 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1178 A ``TransmissionCurve`` that represents the throughput of the
1179 filter itself, to be evaluated in focal-plane coordinates.
1180 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1181 A ``TransmissionCurve`` that represents the throughput of the
1182 sensor itself, to be evaluated in post-assembly trimmed
1183 detector coordinates.
1184 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1185 A ``TransmissionCurve`` that represents the throughput of the
1186 atmosphere, assumed to be spatially constant.
1187 - ``strayLightData`` : `object`
1188 An opaque object containing calibration information for
1189 stray-light correction. If `None`, no correction will be
1190 performed.
1191 - ``illumMaskedImage`` : illumination correction image
1192 (`lsst.afw.image.MaskedImage`)
1194 Raises
1195 ------
1196 NotImplementedError :
1197 Raised if a per-amplifier brighter-fatter kernel is requested by
1198 the configuration.
1199 """
1200 try:
1201 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1202 dateObs = dateObs.toPython().isoformat()
1203 except RuntimeError:
1204 self.log.warning("Unable to identify dateObs for rawExposure.")
1205 dateObs = None
1207 ccd = rawExposure.getDetector()
1208 filterLabel = rawExposure.getFilter()
1209 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1210 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
1211 biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
1212 if self.config.doBias else None)
1213 # immediate=True required for functors and linearizers are functors
1214 # see ticket DM-6515
1215 linearizer = (dataRef.get("linearizer", immediate=True)
1216 if self.doLinearize(ccd) else None)
1217 if linearizer is not None and not isinstance(linearizer, numpy.ndarray):
1218 linearizer.log = self.log
1219 if isinstance(linearizer, numpy.ndarray):
1220 linearizer = linearize.Linearizer(table=linearizer, detector=ccd)
1222 crosstalkCalib = None
1223 if self.config.doCrosstalk:
1224 try:
1225 crosstalkCalib = dataRef.get("crosstalk", immediate=True)
1226 except NoResults:
1227 coeffVector = (self.config.crosstalk.crosstalkValues
1228 if self.config.crosstalk.useConfigCoefficients else None)
1229 crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1230 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1231 if self.config.doCrosstalk else None)
1233 darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
1234 if self.config.doDark else None)
1235 flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName,
1236 dateObs=dateObs)
1237 if self.config.doFlat else None)
1239 brighterFatterKernel = None
1240 brighterFatterGains = None
1241 if self.config.doBrighterFatter is True:
1242 try:
1243 # Use the new-style cp_pipe version of the kernel if it exists
1244 # If using a new-style kernel, always use the self-consistent
1245 # gains, i.e. the ones inside the kernel object itself
1246 brighterFatterKernel = dataRef.get("brighterFatterKernel")
1247 brighterFatterGains = brighterFatterKernel.gain
1248 self.log.info("New style brighter-fatter kernel (brighterFatterKernel) loaded")
1249 except NoResults:
1250 try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
1251 brighterFatterKernel = dataRef.get("bfKernel")
1252 self.log.info("Old style brighter-fatter kernel (bfKernel) loaded")
1253 except NoResults:
1254 brighterFatterKernel = None
1255 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1256 # If the kernel is not an ndarray, it's the cp_pipe version
1257 # so extract the kernel for this detector, or raise an error
1258 if self.config.brighterFatterLevel == 'DETECTOR':
1259 if brighterFatterKernel.detKernels:
1260 brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()]
1261 else:
1262 raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1263 else:
1264 # TODO DM-15631 for implementing this
1265 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1267 defectList = (dataRef.get("defects")
1268 if self.config.doDefect else None)
1269 expId = rawExposure.info.id
1270 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1271 if self.config.doAssembleIsrExposures else None)
1272 if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1273 else pipeBase.Struct(fringes=None))
1275 if self.config.doAttachTransmissionCurve:
1276 opticsTransmission = (dataRef.get("transmission_optics")
1277 if self.config.doUseOpticsTransmission else None)
1278 filterTransmission = (dataRef.get("transmission_filter")
1279 if self.config.doUseFilterTransmission else None)
1280 sensorTransmission = (dataRef.get("transmission_sensor")
1281 if self.config.doUseSensorTransmission else None)
1282 atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1283 if self.config.doUseAtmosphereTransmission else None)
1284 else:
1285 opticsTransmission = None
1286 filterTransmission = None
1287 sensorTransmission = None
1288 atmosphereTransmission = None
1290 if self.config.doStrayLight:
1291 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1292 else:
1293 strayLightData = None
1295 illumMaskedImage = (self.getIsrExposure(dataRef,
1296 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1297 if (self.config.doIlluminationCorrection
1298 and physicalFilter in self.config.illumFilters)
1299 else None)
1301 # Struct should include only kwargs to run()
1302 return pipeBase.Struct(bias=biasExposure,
1303 linearizer=linearizer,
1304 crosstalk=crosstalkCalib,
1305 crosstalkSources=crosstalkSources,
1306 dark=darkExposure,
1307 flat=flatExposure,
1308 bfKernel=brighterFatterKernel,
1309 bfGains=brighterFatterGains,
1310 defects=defectList,
1311 fringes=fringeStruct,
1312 opticsTransmission=opticsTransmission,
1313 filterTransmission=filterTransmission,
1314 sensorTransmission=sensorTransmission,
1315 atmosphereTransmission=atmosphereTransmission,
1316 strayLightData=strayLightData,
1317 illumMaskedImage=illumMaskedImage
1318 )
1320 @timeMethod
1321 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1322 crosstalk=None, crosstalkSources=None,
1323 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1324 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1325 sensorTransmission=None, atmosphereTransmission=None,
1326 detectorNum=None, strayLightData=None, illumMaskedImage=None,
1327 deferredCharge=None, isGen3=False,
1328 ):
1329 """Perform instrument signature removal on an exposure.
1331 Steps included in the ISR processing, in order performed, are:
1332 - saturation and suspect pixel masking
1333 - overscan subtraction
1334 - CCD assembly of individual amplifiers
1335 - bias subtraction
1336 - variance image construction
1337 - linearization of non-linear response
1338 - crosstalk masking
1339 - brighter-fatter correction
1340 - dark subtraction
1341 - fringe correction
1342 - stray light subtraction
1343 - flat correction
1344 - masking of known defects and camera specific features
1345 - vignette calculation
1346 - appending transmission curve and distortion model
1348 Parameters
1349 ----------
1350 ccdExposure : `lsst.afw.image.Exposure`
1351 The raw exposure that is to be run through ISR. The
1352 exposure is modified by this method.
1353 camera : `lsst.afw.cameraGeom.Camera`, optional
1354 The camera geometry for this exposure. Required if
1355 one or more of ``ccdExposure``, ``bias``, ``dark``, or
1356 ``flat`` does not have an associated detector.
1357 bias : `lsst.afw.image.Exposure`, optional
1358 Bias calibration frame.
1359 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1360 Functor for linearization.
1361 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1362 Calibration for crosstalk.
1363 crosstalkSources : `list`, optional
1364 List of possible crosstalk sources.
1365 dark : `lsst.afw.image.Exposure`, optional
1366 Dark calibration frame.
1367 flat : `lsst.afw.image.Exposure`, optional
1368 Flat calibration frame.
1369 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
1370 Photon transfer curve dataset, with, e.g., gains
1371 and read noise.
1372 bfKernel : `numpy.ndarray`, optional
1373 Brighter-fatter kernel.
1374 bfGains : `dict` of `float`, optional
1375 Gains used to override the detector's nominal gains for the
1376 brighter-fatter correction. A dict keyed by amplifier name for
1377 the detector in question.
1378 defects : `lsst.ip.isr.Defects`, optional
1379 List of defects.
1380 fringes : `lsst.pipe.base.Struct`, optional
1381 Struct containing the fringe correction data, with
1382 elements:
1383 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1384 - ``seed``: random seed derived from the ccdExposureId for random
1385 number generator (`uint32`)
1386 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1387 A ``TransmissionCurve`` that represents the throughput of the,
1388 optics, to be evaluated in focal-plane coordinates.
1389 filterTransmission : `lsst.afw.image.TransmissionCurve`
1390 A ``TransmissionCurve`` that represents the throughput of the
1391 filter itself, to be evaluated in focal-plane coordinates.
1392 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1393 A ``TransmissionCurve`` that represents the throughput of the
1394 sensor itself, to be evaluated in post-assembly trimmed detector
1395 coordinates.
1396 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1397 A ``TransmissionCurve`` that represents the throughput of the
1398 atmosphere, assumed to be spatially constant.
1399 detectorNum : `int`, optional
1400 The integer number for the detector to process.
1401 isGen3 : bool, optional
1402 Flag this call to run() as using the Gen3 butler environment.
1403 strayLightData : `object`, optional
1404 Opaque object containing calibration information for stray-light
1405 correction. If `None`, no correction will be performed.
1406 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1407 Illumination correction image.
1409 Returns
1410 -------
1411 result : `lsst.pipe.base.Struct`
1412 Result struct with component:
1413 - ``exposure`` : `afw.image.Exposure`
1414 The fully ISR corrected exposure.
1415 - ``outputExposure`` : `afw.image.Exposure`
1416 An alias for `exposure`
1417 - ``ossThumb`` : `numpy.ndarray`
1418 Thumbnail image of the exposure after overscan subtraction.
1419 - ``flattenedThumb`` : `numpy.ndarray`
1420 Thumbnail image of the exposure after flat-field correction.
1421 - ``outputStatistics`` : ``
1422 Values of the additional statistics calculated.
1424 Raises
1425 ------
1426 RuntimeError
1427 Raised if a configuration option is set to True, but the
1428 required calibration data has not been specified.
1430 Notes
1431 -----
1432 The current processed exposure can be viewed by setting the
1433 appropriate lsstDebug entries in the `debug.display`
1434 dictionary. The names of these entries correspond to some of
1435 the IsrTaskConfig Boolean options, with the value denoting the
1436 frame to use. The exposure is shown inside the matching
1437 option check and after the processing of that step has
1438 finished. The steps with debug points are:
1440 doAssembleCcd
1441 doBias
1442 doCrosstalk
1443 doBrighterFatter
1444 doDark
1445 doFringe
1446 doStrayLight
1447 doFlat
1449 In addition, setting the "postISRCCD" entry displays the
1450 exposure after all ISR processing has finished.
1452 """
1454 if isGen3 is True:
1455 # Gen3 currently cannot automatically do configuration overrides.
1456 # DM-15257 looks to discuss this issue.
1457 # Configure input exposures;
1459 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1460 bias = self.ensureExposure(bias, camera, detectorNum)
1461 dark = self.ensureExposure(dark, camera, detectorNum)
1462 flat = self.ensureExposure(flat, camera, detectorNum)
1463 else:
1464 if isinstance(ccdExposure, ButlerDataRef):
1465 return self.runDataRef(ccdExposure)
1467 ccd = ccdExposure.getDetector()
1468 filterLabel = ccdExposure.getFilter()
1469 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1471 if not ccd:
1472 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1473 ccd = [FakeAmp(ccdExposure, self.config)]
1475 # Validate Input
1476 if self.config.doBias and bias is None:
1477 raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1478 if self.doLinearize(ccd) and linearizer is None:
1479 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1480 if self.config.doBrighterFatter and bfKernel is None:
1481 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1482 if self.config.doDark and dark is None:
1483 raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1484 if self.config.doFlat and flat is None:
1485 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1486 if self.config.doDefect and defects is None:
1487 raise RuntimeError("Must supply defects if config.doDefect=True.")
1488 if (self.config.doFringe and physicalFilter in self.fringe.config.filters
1489 and fringes.fringes is None):
1490 # The `fringes` object needs to be a pipeBase.Struct, as
1491 # we use it as a `dict` for the parameters of
1492 # `FringeTask.run()`. The `fringes.fringes` `list` may
1493 # not be `None` if `doFringe=True`. Otherwise, raise.
1494 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1495 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters
1496 and illumMaskedImage is None):
1497 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1498 if (self.config.doDeferredCharge and deferredCharge is None):
1499 raise RuntimeError("Must supply a deferred charge calibration if config.doDeferredCharge=True.")
1501 # Begin ISR processing.
1502 if self.config.doConvertIntToFloat:
1503 self.log.info("Converting exposure to floating point values.")
1504 ccdExposure = self.convertIntToFloat(ccdExposure)
1506 if self.config.doBias and self.config.doBiasBeforeOverscan:
1507 self.log.info("Applying bias correction.")
1508 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1509 trimToFit=self.config.doTrimToMatchCalib)
1510 self.debugView(ccdExposure, "doBias")
1512 # Amplifier level processing.
1513 overscans = []
1514 for amp in ccd:
1515 # if ccdExposure is one amp,
1516 # check for coverage to prevent performing ops multiple times
1517 if ccdExposure.getBBox().contains(amp.getBBox()):
1518 # Check for fully masked bad amplifiers,
1519 # and generate masks for SUSPECT and SATURATED values.
1520 badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1522 if self.config.doOverscan and not badAmp:
1523 # Overscan correction on amp-by-amp basis.
1524 overscanResults = self.overscanCorrection(ccdExposure, amp)
1525 self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1526 if overscanResults is not None and \
1527 self.config.qa is not None and self.config.qa.saveStats is True:
1528 if isinstance(overscanResults.overscanFit, float):
1529 qaMedian = overscanResults.overscanFit
1530 qaStdev = float("NaN")
1531 else:
1532 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1533 afwMath.MEDIAN | afwMath.STDEVCLIP)
1534 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1535 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1537 self.metadata[f"FIT MEDIAN {amp.getName()}"] = qaMedian
1538 self.metadata[f"FIT STDEV {amp.getName()}"] = qaStdev
1539 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1540 amp.getName(), qaMedian, qaStdev)
1542 # Residuals after overscan correction
1543 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1544 afwMath.MEDIAN | afwMath.STDEVCLIP)
1545 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1546 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1548 self.metadata[f"RESIDUAL MEDIAN {amp.getName()}"] = qaMedianAfter
1549 self.metadata[f"RESIDUAL STDEV {amp.getName()}"] = qaStdevAfter
1550 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f",
1551 amp.getName(), qaMedianAfter, qaStdevAfter)
1553 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1554 else:
1555 if badAmp:
1556 self.log.warning("Amplifier %s is bad.", amp.getName())
1557 overscanResults = None
1559 overscans.append(overscanResults if overscanResults is not None else None)
1560 else:
1561 self.log.info("Skipped OSCAN for %s.", amp.getName())
1563 if self.config.doDeferredCharge:
1564 self.log.info("Applying deferred charge/CTI correction.")
1565 self.deferredChargeCorrection.run(ccdExposure, deferredCharge)
1566 self.debugView(ccdExposure, "doDeferredCharge")
1568 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1569 self.log.info("Applying crosstalk correction.")
1570 self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1571 crosstalkSources=crosstalkSources, camera=camera)
1572 self.debugView(ccdExposure, "doCrosstalk")
1574 if self.config.doAssembleCcd:
1575 self.log.info("Assembling CCD from amplifiers.")
1576 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1578 if self.config.expectWcs and not ccdExposure.getWcs():
1579 self.log.warning("No WCS found in input exposure.")
1580 self.debugView(ccdExposure, "doAssembleCcd")
1582 ossThumb = None
1583 if self.config.qa.doThumbnailOss:
1584 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1586 if self.config.doBias and not self.config.doBiasBeforeOverscan:
1587 self.log.info("Applying bias correction.")
1588 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1589 trimToFit=self.config.doTrimToMatchCalib)
1590 self.debugView(ccdExposure, "doBias")
1592 if self.config.doVariance:
1593 for amp, overscanResults in zip(ccd, overscans):
1594 if ccdExposure.getBBox().contains(amp.getBBox()):
1595 self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1596 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1597 if overscanResults is not None:
1598 self.updateVariance(ampExposure, amp,
1599 overscanImage=overscanResults.overscanImage,
1600 ptcDataset=ptc)
1601 else:
1602 self.updateVariance(ampExposure, amp,
1603 overscanImage=None,
1604 ptcDataset=ptc)
1605 if self.config.qa is not None and self.config.qa.saveStats is True:
1606 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1607 afwMath.MEDIAN | afwMath.STDEVCLIP)
1608 self.metadata[f"ISR VARIANCE {amp.getName()} MEDIAN"] = \
1609 qaStats.getValue(afwMath.MEDIAN)
1610 self.metadata[f"ISR VARIANCE {amp.getName()} STDEV"] = \
1611 qaStats.getValue(afwMath.STDEVCLIP)
1612 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1613 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1614 qaStats.getValue(afwMath.STDEVCLIP))
1615 if self.config.maskNegativeVariance:
1616 self.maskNegativeVariance(ccdExposure)
1618 if self.doLinearize(ccd):
1619 self.log.info("Applying linearizer.")
1620 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1621 detector=ccd, log=self.log)
1623 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1624 self.log.info("Applying crosstalk correction.")
1625 self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1626 crosstalkSources=crosstalkSources, isTrimmed=True)
1627 self.debugView(ccdExposure, "doCrosstalk")
1629 # Masking block. Optionally mask known defects, NAN/inf pixels,
1630 # widen trails, and do anything else the camera needs. Saturated and
1631 # suspect pixels have already been masked.
1632 if self.config.doDefect:
1633 self.log.info("Masking defects.")
1634 self.maskDefect(ccdExposure, defects)
1636 if self.config.numEdgeSuspect > 0:
1637 self.log.info("Masking edges as SUSPECT.")
1638 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1639 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
1641 if self.config.doNanMasking:
1642 self.log.info("Masking non-finite (NAN, inf) value pixels.")
1643 self.maskNan(ccdExposure)
1645 if self.config.doWidenSaturationTrails:
1646 self.log.info("Widening saturation trails.")
1647 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1649 if self.config.doCameraSpecificMasking:
1650 self.log.info("Masking regions for camera specific reasons.")
1651 self.masking.run(ccdExposure)
1653 if self.config.doBrighterFatter:
1654 # We need to apply flats and darks before we can interpolate, and
1655 # we need to interpolate before we do B-F, but we do B-F without
1656 # the flats and darks applied so we can work in units of electrons
1657 # or holes. This context manager applies and then removes the darks
1658 # and flats.
1659 #
1660 # We also do not want to interpolate values here, so operate on
1661 # temporary images so we can apply only the BF-correction and roll
1662 # back the interpolation.
1663 interpExp = ccdExposure.clone()
1664 with self.flatContext(interpExp, flat, dark):
1665 isrFunctions.interpolateFromMask(
1666 maskedImage=interpExp.getMaskedImage(),
1667 fwhm=self.config.fwhm,
1668 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1669 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1670 )
1671 bfExp = interpExp.clone()
1673 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.",
1674 type(bfKernel), type(bfGains))
1675 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1676 self.config.brighterFatterMaxIter,
1677 self.config.brighterFatterThreshold,
1678 self.config.brighterFatterApplyGain,
1679 bfGains)
1680 if bfResults[1] == self.config.brighterFatterMaxIter:
1681 self.log.warning("Brighter-fatter correction did not converge, final difference %f.",
1682 bfResults[0])
1683 else:
1684 self.log.info("Finished brighter-fatter correction in %d iterations.",
1685 bfResults[1])
1686 image = ccdExposure.getMaskedImage().getImage()
1687 bfCorr = bfExp.getMaskedImage().getImage()
1688 bfCorr -= interpExp.getMaskedImage().getImage()
1689 image += bfCorr
1691 # Applying the brighter-fatter correction applies a
1692 # convolution to the science image. At the edges this
1693 # convolution may not have sufficient valid pixels to
1694 # produce a valid correction. Mark pixels within the size
1695 # of the brighter-fatter kernel as EDGE to warn of this
1696 # fact.
1697 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1698 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1699 maskPlane="EDGE")
1701 if self.config.brighterFatterMaskGrowSize > 0:
1702 self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1703 for maskPlane in self.config.brighterFatterMaskListToInterpolate:
1704 isrFunctions.growMasks(ccdExposure.getMask(),
1705 radius=self.config.brighterFatterMaskGrowSize,
1706 maskNameList=maskPlane,
1707 maskValue=maskPlane)
1709 self.debugView(ccdExposure, "doBrighterFatter")
1711 if self.config.doDark:
1712 self.log.info("Applying dark correction.")
1713 self.darkCorrection(ccdExposure, dark)
1714 self.debugView(ccdExposure, "doDark")
1716 if self.config.doFringe and not self.config.fringeAfterFlat:
1717 self.log.info("Applying fringe correction before flat.")
1718 self.fringe.run(ccdExposure, **fringes.getDict())
1719 self.debugView(ccdExposure, "doFringe")
1721 if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1722 self.log.info("Checking strayLight correction.")
1723 self.strayLight.run(ccdExposure, strayLightData)
1724 self.debugView(ccdExposure, "doStrayLight")
1726 if self.config.doFlat:
1727 self.log.info("Applying flat correction.")
1728 self.flatCorrection(ccdExposure, flat)
1729 self.debugView(ccdExposure, "doFlat")
1731 if self.config.doApplyGains:
1732 self.log.info("Applying gain correction instead of flat.")
1733 if self.config.usePtcGains:
1734 self.log.info("Using gains from the Photon Transfer Curve.")
1735 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1736 ptcGains=ptc.gain)
1737 else:
1738 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1740 if self.config.doFringe and self.config.fringeAfterFlat:
1741 self.log.info("Applying fringe correction after flat.")
1742 self.fringe.run(ccdExposure, **fringes.getDict())
1744 if self.config.doVignette:
1745 if self.config.doMaskVignettePolygon:
1746 self.log.info("Constructing, attaching, and masking vignette polygon.")
1747 else:
1748 self.log.info("Constructing and attaching vignette polygon.")
1749 self.vignettePolygon = self.vignette.run(
1750 exposure=ccdExposure, doUpdateMask=self.config.doMaskVignettePolygon,
1751 vignetteValue=self.config.vignetteValue, log=self.log)
1753 if self.config.doAttachTransmissionCurve:
1754 self.log.info("Adding transmission curves.")
1755 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1756 filterTransmission=filterTransmission,
1757 sensorTransmission=sensorTransmission,
1758 atmosphereTransmission=atmosphereTransmission)
1760 flattenedThumb = None
1761 if self.config.qa.doThumbnailFlattened:
1762 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1764 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters:
1765 self.log.info("Performing illumination correction.")
1766 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1767 illumMaskedImage, illumScale=self.config.illumScale,
1768 trimToFit=self.config.doTrimToMatchCalib)
1770 preInterpExp = None
1771 if self.config.doSaveInterpPixels:
1772 preInterpExp = ccdExposure.clone()
1774 # Reset and interpolate bad pixels.
1775 #
1776 # Large contiguous bad regions (which should have the BAD mask
1777 # bit set) should have their values set to the image median.
1778 # This group should include defects and bad amplifiers. As the
1779 # area covered by these defects are large, there's little
1780 # reason to expect that interpolation would provide a more
1781 # useful value.
1782 #
1783 # Smaller defects can be safely interpolated after the larger
1784 # regions have had their pixel values reset. This ensures
1785 # that the remaining defects adjacent to bad amplifiers (as an
1786 # example) do not attempt to interpolate extreme values.
1787 if self.config.doSetBadRegions:
1788 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1789 if badPixelCount > 0:
1790 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1792 if self.config.doInterpolate:
1793 self.log.info("Interpolating masked pixels.")
1794 isrFunctions.interpolateFromMask(
1795 maskedImage=ccdExposure.getMaskedImage(),
1796 fwhm=self.config.fwhm,
1797 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1798 maskNameList=list(self.config.maskListToInterpolate)
1799 )
1801 self.roughZeroPoint(ccdExposure)
1803 # correct for amp offsets within the CCD
1804 if self.config.doAmpOffset:
1805 self.log.info("Correcting amp offsets.")
1806 self.ampOffset.run(ccdExposure)
1808 if self.config.doMeasureBackground:
1809 self.log.info("Measuring background level.")
1810 self.measureBackground(ccdExposure, self.config.qa)
1812 if self.config.qa is not None and self.config.qa.saveStats is True:
1813 for amp in ccd:
1814 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1815 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1816 afwMath.MEDIAN | afwMath.STDEVCLIP)
1817 self.metadata[f"ISR BACKGROUND {amp.getName()} MEDIAN"] = qaStats.getValue(afwMath.MEDIAN)
1818 self.metadata[f"ISR BACKGROUND {amp.getName()} STDEV"] = \
1819 qaStats.getValue(afwMath.STDEVCLIP)
1820 self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1821 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1822 qaStats.getValue(afwMath.STDEVCLIP))
1824 # calculate additional statistics.
1825 outputStatistics = None
1826 if self.config.doCalculateStatistics:
1827 outputStatistics = self.isrStats.run(ccdExposure, overscanResults=overscans,
1828 ptc=ptc).results
1830 self.debugView(ccdExposure, "postISRCCD")
1832 return pipeBase.Struct(
1833 exposure=ccdExposure,
1834 ossThumb=ossThumb,
1835 flattenedThumb=flattenedThumb,
1837 preInterpExposure=preInterpExp,
1838 outputExposure=ccdExposure,
1839 outputOssThumbnail=ossThumb,
1840 outputFlattenedThumbnail=flattenedThumb,
1841 outputStatistics=outputStatistics,
1842 )
1844 @timeMethod
1845 def runDataRef(self, sensorRef):
1846 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1848 This method contains the `CmdLineTask` interface to the ISR
1849 processing. All IO is handled here, freeing the `run()` method
1850 to manage only pixel-level calculations. The steps performed
1851 are:
1852 - Read in necessary detrending/isr/calibration data.
1853 - Process raw exposure in `run()`.
1854 - Persist the ISR-corrected exposure as "postISRCCD" if
1855 config.doWrite=True.
1857 Parameters
1858 ----------
1859 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1860 DataRef of the detector data to be processed
1862 Returns
1863 -------
1864 result : `lsst.pipe.base.Struct`
1865 Result struct with component:
1866 - ``exposure`` : `afw.image.Exposure`
1867 The fully ISR corrected exposure.
1869 Raises
1870 ------
1871 RuntimeError
1872 Raised if a configuration option is set to True, but the
1873 required calibration data does not exist.
1875 """
1876 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1878 ccdExposure = sensorRef.get(self.config.datasetType)
1880 camera = sensorRef.get("camera")
1881 isrData = self.readIsrData(sensorRef, ccdExposure)
1883 result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1885 if self.config.doWrite:
1886 sensorRef.put(result.exposure, "postISRCCD")
1887 if result.preInterpExposure is not None:
1888 sensorRef.put(result.preInterpExposure, "postISRCCD_uninterpolated")
1889 if result.ossThumb is not None:
1890 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1891 if result.flattenedThumb is not None:
1892 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1894 return result
1896 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1897 """Retrieve a calibration dataset for removing instrument signature.
1899 Parameters
1900 ----------
1902 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1903 DataRef of the detector data to find calibration datasets
1904 for.
1905 datasetType : `str`
1906 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1907 dateObs : `str`, optional
1908 Date of the observation. Used to correct butler failures
1909 when using fallback filters.
1910 immediate : `Bool`
1911 If True, disable butler proxies to enable error handling
1912 within this routine.
1914 Returns
1915 -------
1916 exposure : `lsst.afw.image.Exposure`
1917 Requested calibration frame.
1919 Raises
1920 ------
1921 RuntimeError
1922 Raised if no matching calibration frame can be found.
1923 """
1924 try:
1925 exp = dataRef.get(datasetType, immediate=immediate)
1926 except Exception as exc1:
1927 if not self.config.fallbackFilterName:
1928 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1929 try:
1930 if self.config.useFallbackDate and dateObs:
1931 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1932 dateObs=dateObs, immediate=immediate)
1933 else:
1934 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1935 except Exception as exc2:
1936 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1937 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1938 self.log.warning("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1940 if self.config.doAssembleIsrExposures:
1941 exp = self.assembleCcd.assembleCcd(exp)
1942 return exp
1944 def ensureExposure(self, inputExp, camera=None, detectorNum=None):
1945 """Ensure that the data returned by Butler is a fully constructed exp.
1947 ISR requires exposure-level image data for historical reasons, so if we
1948 did not recieve that from Butler, construct it from what we have,
1949 modifying the input in place.
1951 Parameters
1952 ----------
1953 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`,
1954 or `lsst.afw.image.ImageF`
1955 The input data structure obtained from Butler.
1956 camera : `lsst.afw.cameraGeom.camera`, optional
1957 The camera associated with the image. Used to find the appropriate
1958 detector if detector is not already set.
1959 detectorNum : `int`, optional
1960 The detector in the camera to attach, if the detector is not
1961 already set.
1963 Returns
1964 -------
1965 inputExp : `lsst.afw.image.Exposure`
1966 The re-constructed exposure, with appropriate detector parameters.
1968 Raises
1969 ------
1970 TypeError
1971 Raised if the input data cannot be used to construct an exposure.
1972 """
1973 if isinstance(inputExp, afwImage.DecoratedImageU):
1974 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1975 elif isinstance(inputExp, afwImage.ImageF):
1976 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1977 elif isinstance(inputExp, afwImage.MaskedImageF):
1978 inputExp = afwImage.makeExposure(inputExp)
1979 elif isinstance(inputExp, afwImage.Exposure):
1980 pass
1981 elif inputExp is None:
1982 # Assume this will be caught by the setup if it is a problem.
1983 return inputExp
1984 else:
1985 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1986 (type(inputExp), ))
1988 if inputExp.getDetector() is None:
1989 if camera is None or detectorNum is None:
1990 raise RuntimeError('Must supply both a camera and detector number when using exposures '
1991 'without a detector set.')
1992 inputExp.setDetector(camera[detectorNum])
1994 return inputExp
1996 def convertIntToFloat(self, exposure):
1997 """Convert exposure image from uint16 to float.
1999 If the exposure does not need to be converted, the input is
2000 immediately returned. For exposures that are converted to use
2001 floating point pixels, the variance is set to unity and the
2002 mask to zero.
2004 Parameters
2005 ----------
2006 exposure : `lsst.afw.image.Exposure`
2007 The raw exposure to be converted.
2009 Returns
2010 -------
2011 newexposure : `lsst.afw.image.Exposure`
2012 The input ``exposure``, converted to floating point pixels.
2014 Raises
2015 ------
2016 RuntimeError
2017 Raised if the exposure type cannot be converted to float.
2019 """
2020 if isinstance(exposure, afwImage.ExposureF):
2021 # Nothing to be done
2022 self.log.debug("Exposure already of type float.")
2023 return exposure
2024 if not hasattr(exposure, "convertF"):
2025 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
2027 newexposure = exposure.convertF()
2028 newexposure.variance[:] = 1
2029 newexposure.mask[:] = 0x0
2031 return newexposure
2033 def maskAmplifier(self, ccdExposure, amp, defects):
2034 """Identify bad amplifiers, saturated and suspect pixels.
2036 Parameters
2037 ----------
2038 ccdExposure : `lsst.afw.image.Exposure`
2039 Input exposure to be masked.
2040 amp : `lsst.afw.table.AmpInfoCatalog`
2041 Catalog of parameters defining the amplifier on this
2042 exposure to mask.
2043 defects : `lsst.ip.isr.Defects`
2044 List of defects. Used to determine if the entire
2045 amplifier is bad.
2047 Returns
2048 -------
2049 badAmp : `Bool`
2050 If this is true, the entire amplifier area is covered by
2051 defects and unusable.
2053 """
2054 maskedImage = ccdExposure.getMaskedImage()
2056 badAmp = False
2058 # Check if entire amp region is defined as a defect
2059 # NB: need to use amp.getBBox() for correct comparison with current
2060 # defects definition.
2061 if defects is not None:
2062 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
2064 # In the case of a bad amp, we will set mask to "BAD"
2065 # (here use amp.getRawBBox() for correct association with pixels in
2066 # current ccdExposure).
2067 if badAmp:
2068 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
2069 afwImage.PARENT)
2070 maskView = dataView.getMask()
2071 maskView |= maskView.getPlaneBitMask("BAD")
2072 del maskView
2073 return badAmp
2075 # Mask remaining defects after assembleCcd() to allow for defects that
2076 # cross amplifier boundaries. Saturation and suspect pixels can be
2077 # masked now, though.
2078 limits = dict()
2079 if self.config.doSaturation and not badAmp:
2080 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
2081 if self.config.doSuspect and not badAmp:
2082 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
2083 if math.isfinite(self.config.saturation):
2084 limits.update({self.config.saturatedMaskName: self.config.saturation})
2086 for maskName, maskThreshold in limits.items():
2087 if not math.isnan(maskThreshold):
2088 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2089 isrFunctions.makeThresholdMask(
2090 maskedImage=dataView,
2091 threshold=maskThreshold,
2092 growFootprints=0,
2093 maskName=maskName
2094 )
2096 # Determine if we've fully masked this amplifier with SUSPECT and
2097 # SAT pixels.
2098 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
2099 afwImage.PARENT)
2100 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
2101 self.config.suspectMaskName])
2102 if numpy.all(maskView.getArray() & maskVal > 0):
2103 badAmp = True
2104 maskView |= maskView.getPlaneBitMask("BAD")
2106 return badAmp
2108 def overscanCorrection(self, ccdExposure, amp):
2109 """Apply overscan correction in place.
2111 This method does initial pixel rejection of the overscan
2112 region. The overscan can also be optionally segmented to
2113 allow for discontinuous overscan responses to be fit
2114 separately. The actual overscan subtraction is performed by
2115 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
2116 which is called here after the amplifier is preprocessed.
2118 Parameters
2119 ----------
2120 ccdExposure : `lsst.afw.image.Exposure`
2121 Exposure to have overscan correction performed.
2122 amp : `lsst.afw.cameraGeom.Amplifer`
2123 The amplifier to consider while correcting the overscan.
2125 Returns
2126 -------
2127 overscanResults : `lsst.pipe.base.Struct`
2128 Result struct with components:
2129 - ``imageFit`` : scalar or `lsst.afw.image.Image`
2130 Value or fit subtracted from the amplifier image data.
2131 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
2132 Value or fit subtracted from the overscan image data.
2133 - ``overscanImage`` : `lsst.afw.image.Image`
2134 Image of the overscan region with the overscan
2135 correction applied. This quantity is used to estimate
2136 the amplifier read noise empirically.
2138 Raises
2139 ------
2140 RuntimeError
2141 Raised if the ``amp`` does not contain raw pixel information.
2143 See Also
2144 --------
2145 lsst.ip.isr.isrFunctions.overscanCorrection
2146 """
2147 if amp.getRawHorizontalOverscanBBox().isEmpty():
2148 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
2149 return None
2151 statControl = afwMath.StatisticsControl()
2152 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2154 # Determine the bounding boxes
2155 dataBBox = amp.getRawDataBBox()
2156 oscanBBox = amp.getRawHorizontalOverscanBBox()
2157 dx0 = 0
2158 dx1 = 0
2160 prescanBBox = amp.getRawPrescanBBox()
2161 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
2162 dx0 += self.config.overscanNumLeadingColumnsToSkip
2163 dx1 -= self.config.overscanNumTrailingColumnsToSkip
2164 else:
2165 dx0 += self.config.overscanNumTrailingColumnsToSkip
2166 dx1 -= self.config.overscanNumLeadingColumnsToSkip
2168 # Determine if we need to work on subregions of the amplifier
2169 # and overscan.
2170 imageBBoxes = []
2171 overscanBBoxes = []
2173 if ((self.config.overscanBiasJump
2174 and self.config.overscanBiasJumpLocation)
2175 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2176 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
2177 self.config.overscanBiasJumpDevices)):
2178 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
2179 yLower = self.config.overscanBiasJumpLocation
2180 yUpper = dataBBox.getHeight() - yLower
2181 else:
2182 yUpper = self.config.overscanBiasJumpLocation
2183 yLower = dataBBox.getHeight() - yUpper
2185 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2186 lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
2187 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2188 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2189 yLower)))
2191 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
2192 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
2193 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
2194 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2195 yUpper)))
2196 else:
2197 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2198 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
2199 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2200 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2201 oscanBBox.getHeight())))
2203 # Perform overscan correction on subregions, ensuring saturated
2204 # pixels are masked.
2205 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
2206 ampImage = ccdExposure.maskedImage[imageBBox]
2207 overscanImage = ccdExposure.maskedImage[overscanBBox]
2209 overscanArray = overscanImage.image.array
2210 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2211 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2212 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
2214 statControl = afwMath.StatisticsControl()
2215 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2217 overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp)
2219 # If we trimmed columns, we need to restore them.
2220 if dx0 != 0 or dx1 != 0:
2221 fullOverscan = ccdExposure.maskedImage[oscanBBox]
2222 overscanVector = overscanResults.overscanFit.array[:, 0]
2223 overscanModel = afwImage.ImageF(fullOverscan.getDimensions())
2224 overscanModel.array[:, :] = 0.0
2225 overscanModel.array[:, 0:dx0] = overscanVector[:, numpy.newaxis]
2226 overscanModel.array[:, dx1:] = overscanVector[:, numpy.newaxis]
2227 fullOverscanImage = fullOverscan.getImage()
2228 fullOverscanImage -= overscanModel
2229 overscanResults = pipeBase.Struct(imageFit=overscanResults.imageFit,
2230 overscanFit=overscanModel,
2231 overscanImage=fullOverscan,
2232 edgeMask=overscanResults.edgeMask)
2234 # Measure average overscan levels and record them in the metadata.
2235 levelStat = afwMath.MEDIAN
2236 sigmaStat = afwMath.STDEVCLIP
2238 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2239 self.config.qa.flatness.nIter)
2240 metadata = ccdExposure.getMetadata()
2241 ampNum = amp.getName()
2242 # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
2243 if isinstance(overscanResults.overscanFit, float):
2244 metadata[f"ISR_OSCAN_LEVEL{ampNum}"] = overscanResults.overscanFit
2245 metadata[f"ISR_OSCAN_SIGMA{ampNum}"] = 0.0
2246 else:
2247 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2248 metadata[f"ISR_OSCAN_LEVEL{ampNum}"] = stats.getValue(levelStat)
2249 metadata[f"ISR_OSCAN_SIGMA%{ampNum}"] = stats.getValue(sigmaStat)
2251 return overscanResults
2253 def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None):
2254 """Set the variance plane using the gain and read noise
2256 The read noise is calculated from the ``overscanImage`` if the
2257 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2258 the value from the amplifier data is used.
2260 Parameters
2261 ----------
2262 ampExposure : `lsst.afw.image.Exposure`
2263 Exposure to process.
2264 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2265 Amplifier detector data.
2266 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2267 Image of overscan, required only for empirical read noise.
2268 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
2269 PTC dataset containing the gains and read noise.
2272 Raises
2273 ------
2274 RuntimeError
2275 Raised if either ``usePtcGains`` of ``usePtcReadNoise``
2276 are ``True``, but ptcDataset is not provided.
2278 Raised if ```doEmpiricalReadNoise`` is ``True`` but
2279 ``overscanImage`` is ``None``.
2281 See also
2282 --------
2283 lsst.ip.isr.isrFunctions.updateVariance
2284 """
2285 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2286 if self.config.usePtcGains:
2287 if ptcDataset is None:
2288 raise RuntimeError("No ptcDataset provided to use PTC gains.")
2289 else:
2290 gain = ptcDataset.gain[amp.getName()]
2291 self.log.info("Using gain from Photon Transfer Curve.")
2292 else:
2293 gain = amp.getGain()
2295 if math.isnan(gain):
2296 gain = 1.0
2297 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2298 elif gain <= 0:
2299 patchedGain = 1.0
2300 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.",
2301 amp.getName(), gain, patchedGain)
2302 gain = patchedGain
2304 if self.config.doEmpiricalReadNoise and overscanImage is None:
2305 raise RuntimeError("Overscan is none for EmpiricalReadNoise.")
2307 if self.config.doEmpiricalReadNoise and overscanImage is not None:
2308 stats = afwMath.StatisticsControl()
2309 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2310 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2311 self.log.info("Calculated empirical read noise for amp %s: %f.",
2312 amp.getName(), readNoise)
2313 elif self.config.usePtcReadNoise:
2314 if ptcDataset is None:
2315 raise RuntimeError("No ptcDataset provided to use PTC readnoise.")
2316 else:
2317 readNoise = ptcDataset.noise[amp.getName()]
2318 self.log.info("Using read noise from Photon Transfer Curve.")
2319 else:
2320 readNoise = amp.getReadNoise()
2322 isrFunctions.updateVariance(
2323 maskedImage=ampExposure.getMaskedImage(),
2324 gain=gain,
2325 readNoise=readNoise,
2326 )
2328 def maskNegativeVariance(self, exposure):
2329 """Identify and mask pixels with negative variance values.
2331 Parameters
2332 ----------
2333 exposure : `lsst.afw.image.Exposure`
2334 Exposure to process.
2336 See Also
2337 --------
2338 lsst.ip.isr.isrFunctions.updateVariance
2339 """
2340 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
2341 bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
2342 exposure.mask.array[bad] |= maskPlane
2344 def darkCorrection(self, exposure, darkExposure, invert=False):
2345 """Apply dark correction in place.
2347 Parameters
2348 ----------
2349 exposure : `lsst.afw.image.Exposure`
2350 Exposure to process.
2351 darkExposure : `lsst.afw.image.Exposure`
2352 Dark exposure of the same size as ``exposure``.
2353 invert : `Bool`, optional
2354 If True, re-add the dark to an already corrected image.
2356 Raises
2357 ------
2358 RuntimeError
2359 Raised if either ``exposure`` or ``darkExposure`` do not
2360 have their dark time defined.
2362 See Also
2363 --------
2364 lsst.ip.isr.isrFunctions.darkCorrection
2365 """
2366 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2367 if math.isnan(expScale):
2368 raise RuntimeError("Exposure darktime is NAN.")
2369 if darkExposure.getInfo().getVisitInfo() is not None \
2370 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2371 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2372 else:
2373 # DM-17444: darkExposure.getInfo.getVisitInfo() is None
2374 # so getDarkTime() does not exist.
2375 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2376 darkScale = 1.0
2378 isrFunctions.darkCorrection(
2379 maskedImage=exposure.getMaskedImage(),
2380 darkMaskedImage=darkExposure.getMaskedImage(),
2381 expScale=expScale,
2382 darkScale=darkScale,
2383 invert=invert,
2384 trimToFit=self.config.doTrimToMatchCalib
2385 )
2387 def doLinearize(self, detector):
2388 """Check if linearization is needed for the detector cameraGeom.
2390 Checks config.doLinearize and the linearity type of the first
2391 amplifier.
2393 Parameters
2394 ----------
2395 detector : `lsst.afw.cameraGeom.Detector`
2396 Detector to get linearity type from.
2398 Returns
2399 -------
2400 doLinearize : `Bool`
2401 If True, linearization should be performed.
2402 """
2403 return self.config.doLinearize and \
2404 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2406 def flatCorrection(self, exposure, flatExposure, invert=False):
2407 """Apply flat correction in place.
2409 Parameters
2410 ----------
2411 exposure : `lsst.afw.image.Exposure`
2412 Exposure to process.
2413 flatExposure : `lsst.afw.image.Exposure`
2414 Flat exposure of the same size as ``exposure``.
2415 invert : `Bool`, optional
2416 If True, unflatten an already flattened image.
2418 See Also
2419 --------
2420 lsst.ip.isr.isrFunctions.flatCorrection
2421 """
2422 isrFunctions.flatCorrection(
2423 maskedImage=exposure.getMaskedImage(),
2424 flatMaskedImage=flatExposure.getMaskedImage(),
2425 scalingType=self.config.flatScalingType,
2426 userScale=self.config.flatUserScale,
2427 invert=invert,
2428 trimToFit=self.config.doTrimToMatchCalib
2429 )
2431 def saturationDetection(self, exposure, amp):
2432 """Detect and mask saturated pixels in config.saturatedMaskName.
2434 Parameters
2435 ----------
2436 exposure : `lsst.afw.image.Exposure`
2437 Exposure to process. Only the amplifier DataSec is processed.
2438 amp : `lsst.afw.table.AmpInfoCatalog`
2439 Amplifier detector data.
2441 See Also
2442 --------
2443 lsst.ip.isr.isrFunctions.makeThresholdMask
2444 """
2445 if not math.isnan(amp.getSaturation()):
2446 maskedImage = exposure.getMaskedImage()
2447 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2448 isrFunctions.makeThresholdMask(
2449 maskedImage=dataView,
2450 threshold=amp.getSaturation(),
2451 growFootprints=0,
2452 maskName=self.config.saturatedMaskName,
2453 )
2455 def saturationInterpolation(self, exposure):
2456 """Interpolate over saturated pixels, in place.
2458 This method should be called after `saturationDetection`, to
2459 ensure that the saturated pixels have been identified in the
2460 SAT mask. It should also be called after `assembleCcd`, since
2461 saturated regions may cross amplifier boundaries.
2463 Parameters
2464 ----------
2465 exposure : `lsst.afw.image.Exposure`
2466 Exposure to process.
2468 See Also
2469 --------
2470 lsst.ip.isr.isrTask.saturationDetection
2471 lsst.ip.isr.isrFunctions.interpolateFromMask
2472 """
2473 isrFunctions.interpolateFromMask(
2474 maskedImage=exposure.getMaskedImage(),
2475 fwhm=self.config.fwhm,
2476 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2477 maskNameList=list(self.config.saturatedMaskName),
2478 )
2480 def suspectDetection(self, exposure, amp):
2481 """Detect and mask suspect pixels in config.suspectMaskName.
2483 Parameters
2484 ----------
2485 exposure : `lsst.afw.image.Exposure`
2486 Exposure to process. Only the amplifier DataSec is processed.
2487 amp : `lsst.afw.table.AmpInfoCatalog`
2488 Amplifier detector data.
2490 See Also
2491 --------
2492 lsst.ip.isr.isrFunctions.makeThresholdMask
2494 Notes
2495 -----
2496 Suspect pixels are pixels whose value is greater than
2497 amp.getSuspectLevel(). This is intended to indicate pixels that may be
2498 affected by unknown systematics; for example if non-linearity
2499 corrections above a certain level are unstable then that would be a
2500 useful value for suspectLevel. A value of `nan` indicates that no such
2501 level exists and no pixels are to be masked as suspicious.
2502 """
2503 suspectLevel = amp.getSuspectLevel()
2504 if math.isnan(suspectLevel):
2505 return
2507 maskedImage = exposure.getMaskedImage()
2508 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2509 isrFunctions.makeThresholdMask(
2510 maskedImage=dataView,
2511 threshold=suspectLevel,
2512 growFootprints=0,
2513 maskName=self.config.suspectMaskName,
2514 )
2516 def maskDefect(self, exposure, defectBaseList):
2517 """Mask defects using mask plane "BAD", in place.
2519 Parameters
2520 ----------
2521 exposure : `lsst.afw.image.Exposure`
2522 Exposure to process.
2523 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2524 `lsst.afw.image.DefectBase`.
2525 List of defects to mask.
2527 Notes
2528 -----
2529 Call this after CCD assembly, since defects may cross amplifier
2530 boundaries.
2531 """
2532 maskedImage = exposure.getMaskedImage()
2533 if not isinstance(defectBaseList, Defects):
2534 # Promotes DefectBase to Defect
2535 defectList = Defects(defectBaseList)
2536 else:
2537 defectList = defectBaseList
2538 defectList.maskPixels(maskedImage, maskName="BAD")
2540 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2541 """Mask edge pixels with applicable mask plane.
2543 Parameters
2544 ----------
2545 exposure : `lsst.afw.image.Exposure`
2546 Exposure to process.
2547 numEdgePixels : `int`, optional
2548 Number of edge pixels to mask.
2549 maskPlane : `str`, optional
2550 Mask plane name to use.
2551 level : `str`, optional
2552 Level at which to mask edges.
2553 """
2554 maskedImage = exposure.getMaskedImage()
2555 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2557 if numEdgePixels > 0:
2558 if level == 'DETECTOR':
2559 boxes = [maskedImage.getBBox()]
2560 elif level == 'AMP':
2561 boxes = [amp.getBBox() for amp in exposure.getDetector()]
2563 for box in boxes:
2564 # This makes a bbox numEdgeSuspect pixels smaller than the
2565 # image on each side
2566 subImage = maskedImage[box]
2567 box.grow(-numEdgePixels)
2568 # Mask pixels outside box
2569 SourceDetectionTask.setEdgeBits(
2570 subImage,
2571 box,
2572 maskBitMask)
2574 def maskAndInterpolateDefects(self, exposure, defectBaseList):
2575 """Mask and interpolate defects using mask plane "BAD", in place.
2577 Parameters
2578 ----------
2579 exposure : `lsst.afw.image.Exposure`
2580 Exposure to process.
2581 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2582 `lsst.afw.image.DefectBase`.
2583 List of defects to mask and interpolate.
2585 See Also
2586 --------
2587 lsst.ip.isr.isrTask.maskDefect
2588 """
2589 self.maskDefect(exposure, defectBaseList)
2590 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2591 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2592 isrFunctions.interpolateFromMask(
2593 maskedImage=exposure.getMaskedImage(),
2594 fwhm=self.config.fwhm,
2595 growSaturatedFootprints=0,
2596 maskNameList=["BAD"],
2597 )
2599 def maskNan(self, exposure):
2600 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2602 Parameters
2603 ----------
2604 exposure : `lsst.afw.image.Exposure`
2605 Exposure to process.
2607 Notes
2608 -----
2609 We mask over all non-finite values (NaN, inf), including those
2610 that are masked with other bits (because those may or may not be
2611 interpolated over later, and we want to remove all NaN/infs).
2612 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2613 preserve the historical name.
2614 """
2615 maskedImage = exposure.getMaskedImage()
2617 # Find and mask NaNs
2618 maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2619 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2620 numNans = maskNans(maskedImage, maskVal)
2621 self.metadata["NUMNANS"] = numNans
2622 if numNans > 0:
2623 self.log.warning("There were %d unmasked NaNs.", numNans)
2625 def maskAndInterpolateNan(self, exposure):
2626 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2627 in place.
2629 Parameters
2630 ----------
2631 exposure : `lsst.afw.image.Exposure`
2632 Exposure to process.
2634 See Also
2635 --------
2636 lsst.ip.isr.isrTask.maskNan
2637 """
2638 self.maskNan(exposure)
2639 isrFunctions.interpolateFromMask(
2640 maskedImage=exposure.getMaskedImage(),
2641 fwhm=self.config.fwhm,
2642 growSaturatedFootprints=0,
2643 maskNameList=["UNMASKEDNAN"],
2644 )
2646 def measureBackground(self, exposure, IsrQaConfig=None):
2647 """Measure the image background in subgrids, for quality control.
2649 Parameters
2650 ----------
2651 exposure : `lsst.afw.image.Exposure`
2652 Exposure to process.
2653 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2654 Configuration object containing parameters on which background
2655 statistics and subgrids to use.
2656 """
2657 if IsrQaConfig is not None:
2658 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2659 IsrQaConfig.flatness.nIter)
2660 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2661 statsControl.setAndMask(maskVal)
2662 maskedImage = exposure.getMaskedImage()
2663 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2664 skyLevel = stats.getValue(afwMath.MEDIAN)
2665 skySigma = stats.getValue(afwMath.STDEVCLIP)
2666 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2667 metadata = exposure.getMetadata()
2668 metadata["SKYLEVEL"] = skyLevel
2669 metadata["SKYSIGMA"] = skySigma
2671 # calcluating flatlevel over the subgrids
2672 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2673 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2674 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2675 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2676 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2677 skyLevels = numpy.zeros((nX, nY))
2679 for j in range(nY):
2680 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2681 for i in range(nX):
2682 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2684 xLLC = xc - meshXHalf
2685 yLLC = yc - meshYHalf
2686 xURC = xc + meshXHalf - 1
2687 yURC = yc + meshYHalf - 1
2689 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2690 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2692 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2694 good = numpy.where(numpy.isfinite(skyLevels))
2695 skyMedian = numpy.median(skyLevels[good])
2696 flatness = (skyLevels[good] - skyMedian) / skyMedian
2697 flatness_rms = numpy.std(flatness)
2698 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2700 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2701 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2702 nX, nY, flatness_pp, flatness_rms)
2704 metadata["FLATNESS_PP"] = float(flatness_pp)
2705 metadata["FLATNESS_RMS"] = float(flatness_rms)
2706 metadata["FLATNESS_NGRIDS"] = '%dx%d' % (nX, nY)
2707 metadata["FLATNESS_MESHX"] = IsrQaConfig.flatness.meshX
2708 metadata["FLATNESS_MESHY"] = IsrQaConfig.flatness.meshY
2710 def roughZeroPoint(self, exposure):
2711 """Set an approximate magnitude zero point for the exposure.
2713 Parameters
2714 ----------
2715 exposure : `lsst.afw.image.Exposure`
2716 Exposure to process.
2717 """
2718 filterLabel = exposure.getFilter()
2719 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2721 if physicalFilter in self.config.fluxMag0T1:
2722 fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2723 else:
2724 self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter)
2725 fluxMag0 = self.config.defaultFluxMag0T1
2727 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2728 if not expTime > 0: # handle NaN as well as <= 0
2729 self.log.warning("Non-positive exposure time; skipping rough zero point.")
2730 return
2732 self.log.info("Setting rough magnitude zero point for filter %s: %f",
2733 physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2734 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2736 @contextmanager
2737 def flatContext(self, exp, flat, dark=None):
2738 """Context manager that applies and removes flats and darks,
2739 if the task is configured to apply them.
2741 Parameters
2742 ----------
2743 exp : `lsst.afw.image.Exposure`
2744 Exposure to process.
2745 flat : `lsst.afw.image.Exposure`
2746 Flat exposure the same size as ``exp``.
2747 dark : `lsst.afw.image.Exposure`, optional
2748 Dark exposure the same size as ``exp``.
2750 Yields
2751 ------
2752 exp : `lsst.afw.image.Exposure`
2753 The flat and dark corrected exposure.
2754 """
2755 if self.config.doDark and dark is not None:
2756 self.darkCorrection(exp, dark)
2757 if self.config.doFlat:
2758 self.flatCorrection(exp, flat)
2759 try:
2760 yield exp
2761 finally:
2762 if self.config.doFlat:
2763 self.flatCorrection(exp, flat, invert=True)
2764 if self.config.doDark and dark is not None:
2765 self.darkCorrection(exp, dark, invert=True)
2767 def debugView(self, exposure, stepname):
2768 """Utility function to examine ISR exposure at different stages.
2770 Parameters
2771 ----------
2772 exposure : `lsst.afw.image.Exposure`
2773 Exposure to view.
2774 stepname : `str`
2775 State of processing to view.
2776 """
2777 frame = getDebugFrame(self._display, stepname)
2778 if frame:
2779 display = getDisplay(frame)
2780 display.scale('asinh', 'zscale')
2781 display.mtv(exposure)
2782 prompt = "Press Enter to continue [c]... "
2783 while True:
2784 ans = input(prompt).lower()
2785 if ans in ("", "c",):
2786 break
2789class FakeAmp(object):
2790 """A Detector-like object that supports returning gain and saturation level
2792 This is used when the input exposure does not have a detector.
2794 Parameters
2795 ----------
2796 exposure : `lsst.afw.image.Exposure`
2797 Exposure to generate a fake amplifier for.
2798 config : `lsst.ip.isr.isrTaskConfig`
2799 Configuration to apply to the fake amplifier.
2800 """
2802 def __init__(self, exposure, config):
2803 self._bbox = exposure.getBBox(afwImage.LOCAL)
2804 self._RawHorizontalOverscanBBox = lsst.geom.Box2I()
2805 self._gain = config.gain
2806 self._readNoise = config.readNoise
2807 self._saturation = config.saturation
2809 def getBBox(self):
2810 return self._bbox
2812 def getRawBBox(self):
2813 return self._bbox
2815 def getRawHorizontalOverscanBBox(self):
2816 return self._RawHorizontalOverscanBBox
2818 def getGain(self):
2819 return self._gain
2821 def getReadNoise(self):
2822 return self._readNoise
2824 def getSaturation(self):
2825 return self._saturation
2827 def getSuspectLevel(self):
2828 return float("NaN")
2831class RunIsrConfig(pexConfig.Config):
2832 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2835class RunIsrTask(pipeBase.CmdLineTask):
2836 """Task to wrap the default IsrTask to allow it to be retargeted.
2838 The standard IsrTask can be called directly from a command line
2839 program, but doing so removes the ability of the task to be
2840 retargeted. As most cameras override some set of the IsrTask
2841 methods, this would remove those data-specific methods in the
2842 output post-ISR images. This wrapping class fixes the issue,
2843 allowing identical post-ISR images to be generated by both the
2844 processCcd and isrTask code.
2845 """
2846 ConfigClass = RunIsrConfig
2847 _DefaultName = "runIsr"
2849 def __init__(self, *args, **kwargs):
2850 super().__init__(*args, **kwargs)
2851 self.makeSubtask("isr")
2853 def runDataRef(self, dataRef):
2854 """
2855 Parameters
2856 ----------
2857 dataRef : `lsst.daf.persistence.ButlerDataRef`
2858 data reference of the detector data to be processed
2860 Returns
2861 -------
2862 result : `pipeBase.Struct`
2863 Result struct with component:
2865 - exposure : `lsst.afw.image.Exposure`
2866 Post-ISR processed exposure.
2867 """
2868 return self.isr.runDataRef(dataRef)