Coverage for python/lsst/ip/isr/isrTask.py : 16%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import math
23import numpy
25import lsst.geom
26import lsst.afw.image as afwImage
27import lsst.afw.math as afwMath
28import lsst.pex.config as pexConfig
29import lsst.pipe.base as pipeBase
30import lsst.pipe.base.connectionTypes as cT
32from contextlib import contextmanager
33from lsstDebug import getDebugFrame
35from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36 ReadoutCorner)
37from lsst.afw.display import getDisplay
38from lsst.afw.geom import Polygon
39from lsst.daf.persistence import ButlerDataRef
40from lsst.daf.persistence.butler import NoResults
41from lsst.meas.algorithms.detection import SourceDetectionTask
43from . import isrFunctions
44from . import isrQa
45from . import linearize
46from .defects import Defects
48from .assembleCcdTask import AssembleCcdTask
49from .crosstalk import CrosstalkTask, CrosstalkCalib
50from .fringe import FringeTask
51from .isr import maskNans
52from .masking import MaskingTask
53from .overscan import OverscanCorrectionTask
54from .straylight import StrayLightTask
55from .vignette import VignetteTask
56from lsst.daf.butler import DimensionGraph
59__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
62def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections):
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
67 populated.
69 This will be unused until DM-25348 resolves the quantum graph
70 generation issue.
72 Parameters
73 ----------
74 datasetType : `str`
75 Dataset to lookup.
76 registry : `lsst.daf.butler.Registry`
77 Butler registry to query.
78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79 Data id to transform to identify crosstalkSources. The
80 ``detector`` entry will be stripped.
81 collections : `lsst.daf.butler.CollectionSearch`
82 Collections to search through.
84 Returns
85 -------
86 results : `list` [`lsst.daf.butler.DatasetRef`]
87 List of datasets that match the query that will be used as
88 crosstalkSources.
89 """
90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"]))
91 results = list(registry.queryDatasets(datasetType,
92 collections=collections,
93 dataId=newDataId,
94 findFirst=True,
95 ).expanded())
96 return results
99class IsrTaskConnections(pipeBase.PipelineTaskConnections,
100 dimensions={"instrument", "exposure", "detector"},
101 defaultTemplates={}):
102 ccdExposure = cT.Input(
103 name="raw",
104 doc="Input exposure to process.",
105 storageClass="Exposure",
106 dimensions=["instrument", "exposure", "detector"],
107 )
108 camera = cT.PrerequisiteInput(
109 name="camera",
110 storageClass="Camera",
111 doc="Input camera to construct complete exposures.",
112 dimensions=["instrument"],
113 isCalibration=True,
114 )
116 crosstalk = cT.PrerequisiteInput(
117 name="crosstalk",
118 doc="Input crosstalk object",
119 storageClass="CrosstalkCalib",
120 dimensions=["instrument", "detector"],
121 isCalibration=True,
122 minimum=0, # can fall back to cameraGeom
123 )
124 # TODO: DM-25348. This does not work yet to correctly load
125 # possible crosstalk sources.
126 crosstalkSources = cT.PrerequisiteInput(
127 name="isrOverscanCorrected",
128 doc="Overscan corrected input images.",
129 storageClass="Exposure",
130 dimensions=["instrument", "exposure", "detector"],
131 deferLoad=True,
132 multiple=True,
133 lookupFunction=crosstalkSourceLookup,
134 minimum=0, # not needed for all instruments, no config to control this
135 )
136 bias = cT.PrerequisiteInput(
137 name="bias",
138 doc="Input bias calibration.",
139 storageClass="ExposureF",
140 dimensions=["instrument", "detector"],
141 isCalibration=True,
142 )
143 dark = cT.PrerequisiteInput(
144 name='dark',
145 doc="Input dark calibration.",
146 storageClass="ExposureF",
147 dimensions=["instrument", "detector"],
148 isCalibration=True,
149 )
150 flat = cT.PrerequisiteInput(
151 name="flat",
152 doc="Input flat calibration.",
153 storageClass="ExposureF",
154 dimensions=["instrument", "physical_filter", "detector"],
155 isCalibration=True,
156 )
157 ptc = cT.PrerequisiteInput(
158 name="ptc",
159 doc="Input Photon Transfer Curve dataset",
160 storageClass="PhotonTransferCurveDataset",
161 dimensions=["instrument", "detector"],
162 isCalibration=True,
163 )
164 fringes = cT.PrerequisiteInput(
165 name="fringe",
166 doc="Input fringe calibration.",
167 storageClass="ExposureF",
168 dimensions=["instrument", "physical_filter", "detector"],
169 isCalibration=True,
170 minimum=0, # only needed for some bands, even when enabled
171 )
172 strayLightData = cT.PrerequisiteInput(
173 name='yBackground',
174 doc="Input stray light calibration.",
175 storageClass="StrayLightData",
176 dimensions=["instrument", "physical_filter", "detector"],
177 deferLoad=True,
178 isCalibration=True,
179 minimum=0, # only needed for some bands, even when enabled
180 )
181 bfKernel = cT.PrerequisiteInput(
182 name='bfKernel',
183 doc="Input brighter-fatter kernel.",
184 storageClass="NumpyArray",
185 dimensions=["instrument"],
186 isCalibration=True,
187 minimum=0, # can use either bfKernel or newBFKernel
188 )
189 newBFKernel = cT.PrerequisiteInput(
190 name='brighterFatterKernel',
191 doc="Newer complete kernel + gain solutions.",
192 storageClass="BrighterFatterKernel",
193 dimensions=["instrument", "detector"],
194 isCalibration=True,
195 minimum=0, # can use either bfKernel or newBFKernel
196 )
197 defects = cT.PrerequisiteInput(
198 name='defects',
199 doc="Input defect tables.",
200 storageClass="Defects",
201 dimensions=["instrument", "detector"],
202 isCalibration=True,
203 )
204 linearizer = cT.PrerequisiteInput(
205 name='linearizer',
206 storageClass="Linearizer",
207 doc="Linearity correction calibration.",
208 dimensions=["instrument", "detector"],
209 isCalibration=True,
210 minimum=0, # can fall back to cameraGeom
211 )
212 opticsTransmission = cT.PrerequisiteInput(
213 name="transmission_optics",
214 storageClass="TransmissionCurve",
215 doc="Transmission curve due to the optics.",
216 dimensions=["instrument"],
217 isCalibration=True,
218 )
219 filterTransmission = cT.PrerequisiteInput(
220 name="transmission_filter",
221 storageClass="TransmissionCurve",
222 doc="Transmission curve due to the filter.",
223 dimensions=["instrument", "physical_filter"],
224 isCalibration=True,
225 )
226 sensorTransmission = cT.PrerequisiteInput(
227 name="transmission_sensor",
228 storageClass="TransmissionCurve",
229 doc="Transmission curve due to the sensor.",
230 dimensions=["instrument", "detector"],
231 isCalibration=True,
232 )
233 atmosphereTransmission = cT.PrerequisiteInput(
234 name="transmission_atmosphere",
235 storageClass="TransmissionCurve",
236 doc="Transmission curve due to the atmosphere.",
237 dimensions=["instrument"],
238 isCalibration=True,
239 )
240 illumMaskedImage = cT.PrerequisiteInput(
241 name="illum",
242 doc="Input illumination correction.",
243 storageClass="MaskedImageF",
244 dimensions=["instrument", "physical_filter", "detector"],
245 isCalibration=True,
246 )
248 outputExposure = cT.Output(
249 name='postISRCCD',
250 doc="Output ISR processed exposure.",
251 storageClass="Exposure",
252 dimensions=["instrument", "exposure", "detector"],
253 )
254 preInterpExposure = cT.Output(
255 name='preInterpISRCCD',
256 doc="Output ISR processed exposure, with pixels left uninterpolated.",
257 storageClass="ExposureF",
258 dimensions=["instrument", "exposure", "detector"],
259 )
260 outputOssThumbnail = cT.Output(
261 name="OssThumb",
262 doc="Output Overscan-subtracted thumbnail image.",
263 storageClass="Thumbnail",
264 dimensions=["instrument", "exposure", "detector"],
265 )
266 outputFlattenedThumbnail = cT.Output(
267 name="FlattenedThumb",
268 doc="Output flat-corrected thumbnail image.",
269 storageClass="Thumbnail",
270 dimensions=["instrument", "exposure", "detector"],
271 )
273 def __init__(self, *, config=None):
274 super().__init__(config=config)
276 if config.doBias is not True:
277 self.prerequisiteInputs.discard("bias")
278 if config.doLinearize is not True:
279 self.prerequisiteInputs.discard("linearizer")
280 if config.doCrosstalk is not True:
281 self.inputs.discard("crosstalkSources")
282 self.prerequisiteInputs.discard("crosstalk")
283 if config.doBrighterFatter is not True:
284 self.prerequisiteInputs.discard("bfKernel")
285 self.prerequisiteInputs.discard("newBFKernel")
286 if config.doDefect is not True:
287 self.prerequisiteInputs.discard("defects")
288 if config.doDark is not True:
289 self.prerequisiteInputs.discard("dark")
290 if config.doFlat is not True:
291 self.prerequisiteInputs.discard("flat")
292 if config.doFringe is not True:
293 self.prerequisiteInputs.discard("fringe")
294 if config.doStrayLight is not True:
295 self.prerequisiteInputs.discard("strayLightData")
296 if config.usePtcGains is not True and config.usePtcReadNoise is not True:
297 self.prerequisiteInputs.discard("ptc")
298 if config.doAttachTransmissionCurve is not True:
299 self.prerequisiteInputs.discard("opticsTransmission")
300 self.prerequisiteInputs.discard("filterTransmission")
301 self.prerequisiteInputs.discard("sensorTransmission")
302 self.prerequisiteInputs.discard("atmosphereTransmission")
303 if config.doUseOpticsTransmission is not True:
304 self.prerequisiteInputs.discard("opticsTransmission")
305 if config.doUseFilterTransmission is not True:
306 self.prerequisiteInputs.discard("filterTransmission")
307 if config.doUseSensorTransmission is not True:
308 self.prerequisiteInputs.discard("sensorTransmission")
309 if config.doUseAtmosphereTransmission is not True:
310 self.prerequisiteInputs.discard("atmosphereTransmission")
311 if config.doIlluminationCorrection is not True:
312 self.prerequisiteInputs.discard("illumMaskedImage")
314 if config.doWrite is not True:
315 self.outputs.discard("outputExposure")
316 self.outputs.discard("preInterpExposure")
317 self.outputs.discard("outputFlattenedThumbnail")
318 self.outputs.discard("outputOssThumbnail")
319 if config.doSaveInterpPixels is not True:
320 self.outputs.discard("preInterpExposure")
321 if config.qa.doThumbnailOss is not True:
322 self.outputs.discard("outputOssThumbnail")
323 if config.qa.doThumbnailFlattened is not True:
324 self.outputs.discard("outputFlattenedThumbnail")
327class IsrTaskConfig(pipeBase.PipelineTaskConfig,
328 pipelineConnections=IsrTaskConnections):
329 """Configuration parameters for IsrTask.
331 Items are grouped in the order in which they are executed by the task.
332 """
333 datasetType = pexConfig.Field(
334 dtype=str,
335 doc="Dataset type for input data; users will typically leave this alone, "
336 "but camera-specific ISR tasks will override it",
337 default="raw",
338 )
340 fallbackFilterName = pexConfig.Field(
341 dtype=str,
342 doc="Fallback default filter name for calibrations.",
343 optional=True
344 )
345 useFallbackDate = pexConfig.Field(
346 dtype=bool,
347 doc="Pass observation date when using fallback filter.",
348 default=False,
349 )
350 expectWcs = pexConfig.Field(
351 dtype=bool,
352 default=True,
353 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
354 )
355 fwhm = pexConfig.Field(
356 dtype=float,
357 doc="FWHM of PSF in arcseconds.",
358 default=1.0,
359 )
360 qa = pexConfig.ConfigField(
361 dtype=isrQa.IsrQaConfig,
362 doc="QA related configuration options.",
363 )
365 # Image conversion configuration
366 doConvertIntToFloat = pexConfig.Field(
367 dtype=bool,
368 doc="Convert integer raw images to floating point values?",
369 default=True,
370 )
372 # Saturated pixel handling.
373 doSaturation = pexConfig.Field(
374 dtype=bool,
375 doc="Mask saturated pixels? NB: this is totally independent of the"
376 " interpolation option - this is ONLY setting the bits in the mask."
377 " To have them interpolated make sure doSaturationInterpolation=True",
378 default=True,
379 )
380 saturatedMaskName = pexConfig.Field(
381 dtype=str,
382 doc="Name of mask plane to use in saturation detection and interpolation",
383 default="SAT",
384 )
385 saturation = pexConfig.Field(
386 dtype=float,
387 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
388 default=float("NaN"),
389 )
390 growSaturationFootprintSize = pexConfig.Field(
391 dtype=int,
392 doc="Number of pixels by which to grow the saturation footprints",
393 default=1,
394 )
396 # Suspect pixel handling.
397 doSuspect = pexConfig.Field(
398 dtype=bool,
399 doc="Mask suspect pixels?",
400 default=False,
401 )
402 suspectMaskName = pexConfig.Field(
403 dtype=str,
404 doc="Name of mask plane to use for suspect pixels",
405 default="SUSPECT",
406 )
407 numEdgeSuspect = pexConfig.Field(
408 dtype=int,
409 doc="Number of edge pixels to be flagged as untrustworthy.",
410 default=0,
411 )
412 edgeMaskLevel = pexConfig.ChoiceField(
413 dtype=str,
414 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
415 default="DETECTOR",
416 allowed={
417 'DETECTOR': 'Mask only the edges of the full detector.',
418 'AMP': 'Mask edges of each amplifier.',
419 },
420 )
422 # Initial masking options.
423 doSetBadRegions = pexConfig.Field(
424 dtype=bool,
425 doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
426 default=True,
427 )
428 badStatistic = pexConfig.ChoiceField(
429 dtype=str,
430 doc="How to estimate the average value for BAD regions.",
431 default='MEANCLIP',
432 allowed={
433 "MEANCLIP": "Correct using the (clipped) mean of good data",
434 "MEDIAN": "Correct using the median of the good data",
435 },
436 )
438 # Overscan subtraction configuration.
439 doOverscan = pexConfig.Field(
440 dtype=bool,
441 doc="Do overscan subtraction?",
442 default=True,
443 )
444 overscan = pexConfig.ConfigurableField(
445 target=OverscanCorrectionTask,
446 doc="Overscan subtraction task for image segments.",
447 )
449 overscanFitType = pexConfig.ChoiceField(
450 dtype=str,
451 doc="The method for fitting the overscan bias level.",
452 default='MEDIAN',
453 allowed={
454 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
455 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
456 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
457 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
458 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
459 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
460 "MEAN": "Correct using the mean of the overscan region",
461 "MEANCLIP": "Correct using a clipped mean of the overscan region",
462 "MEDIAN": "Correct using the median of the overscan region",
463 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
464 },
465 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
466 " This option will no longer be used, and will be removed after v20.")
467 )
468 overscanOrder = pexConfig.Field(
469 dtype=int,
470 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, "
471 "or number of spline knots if overscan fit type is a spline."),
472 default=1,
473 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
474 " This option will no longer be used, and will be removed after v20.")
475 )
476 overscanNumSigmaClip = pexConfig.Field(
477 dtype=float,
478 doc="Rejection threshold (sigma) for collapsing overscan before fit",
479 default=3.0,
480 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
481 " This option will no longer be used, and will be removed after v20.")
482 )
483 overscanIsInt = pexConfig.Field(
484 dtype=bool,
485 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
486 " and overscan.FitType=MEDIAN_PER_ROW.",
487 default=True,
488 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
489 " This option will no longer be used, and will be removed after v20.")
490 )
491 # These options do not get deprecated, as they define how we slice up the image data.
492 overscanNumLeadingColumnsToSkip = pexConfig.Field(
493 dtype=int,
494 doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
495 default=0,
496 )
497 overscanNumTrailingColumnsToSkip = pexConfig.Field(
498 dtype=int,
499 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
500 default=0,
501 )
502 overscanMaxDev = pexConfig.Field( 502 ↛ exitline 502 didn't jump to the function exit
503 dtype=float,
504 doc="Maximum deviation from the median for overscan",
505 default=1000.0, check=lambda x: x > 0
506 )
507 overscanBiasJump = pexConfig.Field(
508 dtype=bool,
509 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
510 default=False,
511 )
512 overscanBiasJumpKeyword = pexConfig.Field(
513 dtype=str,
514 doc="Header keyword containing information about devices.",
515 default="NO_SUCH_KEY",
516 )
517 overscanBiasJumpDevices = pexConfig.ListField(
518 dtype=str,
519 doc="List of devices that need piecewise overscan correction.",
520 default=(),
521 )
522 overscanBiasJumpLocation = pexConfig.Field(
523 dtype=int,
524 doc="Location of bias jump along y-axis.",
525 default=0,
526 )
528 # Amplifier to CCD assembly configuration
529 doAssembleCcd = pexConfig.Field(
530 dtype=bool,
531 default=True,
532 doc="Assemble amp-level exposures into a ccd-level exposure?"
533 )
534 assembleCcd = pexConfig.ConfigurableField(
535 target=AssembleCcdTask,
536 doc="CCD assembly task",
537 )
539 # General calibration configuration.
540 doAssembleIsrExposures = pexConfig.Field(
541 dtype=bool,
542 default=False,
543 doc="Assemble amp-level calibration exposures into ccd-level exposure?"
544 )
545 doTrimToMatchCalib = pexConfig.Field(
546 dtype=bool,
547 default=False,
548 doc="Trim raw data to match calibration bounding boxes?"
549 )
551 # Bias subtraction.
552 doBias = pexConfig.Field(
553 dtype=bool,
554 doc="Apply bias frame correction?",
555 default=True,
556 )
557 biasDataProductName = pexConfig.Field(
558 dtype=str,
559 doc="Name of the bias data product",
560 default="bias",
561 )
562 doBiasBeforeOverscan = pexConfig.Field(
563 dtype=bool,
564 doc="Reverse order of overscan and bias correction.",
565 default=False
566 )
568 # Variance construction
569 doVariance = pexConfig.Field(
570 dtype=bool,
571 doc="Calculate variance?",
572 default=True
573 )
574 gain = pexConfig.Field(
575 dtype=float,
576 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
577 default=float("NaN"),
578 )
579 readNoise = pexConfig.Field(
580 dtype=float,
581 doc="The read noise to use if no Detector is present in the Exposure",
582 default=0.0,
583 )
584 doEmpiricalReadNoise = pexConfig.Field(
585 dtype=bool,
586 default=False,
587 doc="Calculate empirical read noise instead of value from AmpInfo data?"
588 )
589 usePtcReadNoise = pexConfig.Field(
590 dtype=bool,
591 default=False,
592 doc="Use readnoise values from the Photon Transfer Curve?"
593 )
594 maskNegativeVariance = pexConfig.Field(
595 dtype=bool,
596 default=True,
597 doc="Mask pixels that claim a negative variance? This likely indicates a failure "
598 "in the measurement of the overscan at an edge due to the data falling off faster "
599 "than the overscan model can account for it."
600 )
601 negativeVarianceMaskName = pexConfig.Field(
602 dtype=str,
603 default="BAD",
604 doc="Mask plane to use to mark pixels with negative variance, if `maskNegativeVariance` is True.",
605 )
606 # Linearization.
607 doLinearize = pexConfig.Field(
608 dtype=bool,
609 doc="Correct for nonlinearity of the detector's response?",
610 default=True,
611 )
613 # Crosstalk.
614 doCrosstalk = pexConfig.Field(
615 dtype=bool,
616 doc="Apply intra-CCD crosstalk correction?",
617 default=False,
618 )
619 doCrosstalkBeforeAssemble = pexConfig.Field(
620 dtype=bool,
621 doc="Apply crosstalk correction before CCD assembly, and before trimming?",
622 default=False,
623 )
624 crosstalk = pexConfig.ConfigurableField(
625 target=CrosstalkTask,
626 doc="Intra-CCD crosstalk correction",
627 )
629 # Masking options.
630 doDefect = pexConfig.Field(
631 dtype=bool,
632 doc="Apply correction for CCD defects, e.g. hot pixels?",
633 default=True,
634 )
635 doNanMasking = pexConfig.Field(
636 dtype=bool,
637 doc="Mask non-finite (NAN, inf) pixels?",
638 default=True,
639 )
640 doWidenSaturationTrails = pexConfig.Field(
641 dtype=bool,
642 doc="Widen bleed trails based on their width?",
643 default=True
644 )
646 # Brighter-Fatter correction.
647 doBrighterFatter = pexConfig.Field(
648 dtype=bool,
649 default=False,
650 doc="Apply the brighter-fatter correction?"
651 )
652 brighterFatterLevel = pexConfig.ChoiceField(
653 dtype=str,
654 default="DETECTOR",
655 doc="The level at which to correct for brighter-fatter.",
656 allowed={
657 "AMP": "Every amplifier treated separately.",
658 "DETECTOR": "One kernel per detector",
659 }
660 )
661 brighterFatterMaxIter = pexConfig.Field(
662 dtype=int,
663 default=10,
664 doc="Maximum number of iterations for the brighter-fatter correction"
665 )
666 brighterFatterThreshold = pexConfig.Field(
667 dtype=float,
668 default=1000,
669 doc="Threshold used to stop iterating the brighter-fatter correction. It is the "
670 "absolute value of the difference between the current corrected image and the one "
671 "from the previous iteration summed over all the pixels."
672 )
673 brighterFatterApplyGain = pexConfig.Field(
674 dtype=bool,
675 default=True,
676 doc="Should the gain be applied when applying the brighter-fatter correction?"
677 )
678 brighterFatterMaskListToInterpolate = pexConfig.ListField(
679 dtype=str,
680 doc="List of mask planes that should be interpolated over when applying the brighter-fatter "
681 "correction.",
682 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"],
683 )
684 brighterFatterMaskGrowSize = pexConfig.Field(
685 dtype=int,
686 default=0,
687 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
688 "when brighter-fatter correction is applied."
689 )
691 # Dark subtraction.
692 doDark = pexConfig.Field(
693 dtype=bool,
694 doc="Apply dark frame correction?",
695 default=True,
696 )
697 darkDataProductName = pexConfig.Field(
698 dtype=str,
699 doc="Name of the dark data product",
700 default="dark",
701 )
703 # Camera-specific stray light removal.
704 doStrayLight = pexConfig.Field(
705 dtype=bool,
706 doc="Subtract stray light in the y-band (due to encoder LEDs)?",
707 default=False,
708 )
709 strayLight = pexConfig.ConfigurableField(
710 target=StrayLightTask,
711 doc="y-band stray light correction"
712 )
714 # Flat correction.
715 doFlat = pexConfig.Field(
716 dtype=bool,
717 doc="Apply flat field correction?",
718 default=True,
719 )
720 flatDataProductName = pexConfig.Field(
721 dtype=str,
722 doc="Name of the flat data product",
723 default="flat",
724 )
725 flatScalingType = pexConfig.ChoiceField(
726 dtype=str,
727 doc="The method for scaling the flat on the fly.",
728 default='USER',
729 allowed={
730 "USER": "Scale by flatUserScale",
731 "MEAN": "Scale by the inverse of the mean",
732 "MEDIAN": "Scale by the inverse of the median",
733 },
734 )
735 flatUserScale = pexConfig.Field(
736 dtype=float,
737 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
738 default=1.0,
739 )
740 doTweakFlat = pexConfig.Field(
741 dtype=bool,
742 doc="Tweak flats to match observed amplifier ratios?",
743 default=False
744 )
746 # Amplifier normalization based on gains instead of using flats configuration.
747 doApplyGains = pexConfig.Field(
748 dtype=bool,
749 doc="Correct the amplifiers for their gains instead of applying flat correction",
750 default=False,
751 )
752 usePtcGains = pexConfig.Field(
753 dtype=bool,
754 doc="Use the gain values from the Photon Transfer Curve?",
755 default=False,
756 )
757 normalizeGains = pexConfig.Field(
758 dtype=bool,
759 doc="Normalize all the amplifiers in each CCD to have the same median value.",
760 default=False,
761 )
763 # Fringe correction.
764 doFringe = pexConfig.Field(
765 dtype=bool,
766 doc="Apply fringe correction?",
767 default=True,
768 )
769 fringe = pexConfig.ConfigurableField(
770 target=FringeTask,
771 doc="Fringe subtraction task",
772 )
773 fringeAfterFlat = pexConfig.Field(
774 dtype=bool,
775 doc="Do fringe subtraction after flat-fielding?",
776 default=True,
777 )
779 # Initial CCD-level background statistics options.
780 doMeasureBackground = pexConfig.Field(
781 dtype=bool,
782 doc="Measure the background level on the reduced image?",
783 default=False,
784 )
786 # Camera-specific masking configuration.
787 doCameraSpecificMasking = pexConfig.Field(
788 dtype=bool,
789 doc="Mask camera-specific bad regions?",
790 default=False,
791 )
792 masking = pexConfig.ConfigurableField(
793 target=MaskingTask,
794 doc="Masking task."
795 )
797 # Interpolation options.
799 doInterpolate = pexConfig.Field(
800 dtype=bool,
801 doc="Interpolate masked pixels?",
802 default=True,
803 )
804 doSaturationInterpolation = pexConfig.Field(
805 dtype=bool,
806 doc="Perform interpolation over pixels masked as saturated?"
807 " NB: This is independent of doSaturation; if that is False this plane"
808 " will likely be blank, resulting in a no-op here.",
809 default=True,
810 )
811 doNanInterpolation = pexConfig.Field(
812 dtype=bool,
813 doc="Perform interpolation over pixels masked as NaN?"
814 " NB: This is independent of doNanMasking; if that is False this plane"
815 " will likely be blank, resulting in a no-op here.",
816 default=True,
817 )
818 doNanInterpAfterFlat = pexConfig.Field(
819 dtype=bool,
820 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
821 "also have to interpolate them before flat-fielding."),
822 default=False,
823 )
824 maskListToInterpolate = pexConfig.ListField(
825 dtype=str,
826 doc="List of mask planes that should be interpolated.",
827 default=['SAT', 'BAD'],
828 )
829 doSaveInterpPixels = pexConfig.Field(
830 dtype=bool,
831 doc="Save a copy of the pre-interpolated pixel values?",
832 default=False,
833 )
835 # Default photometric calibration options.
836 fluxMag0T1 = pexConfig.DictField(
837 keytype=str,
838 itemtype=float,
839 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
840 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
841 ))
842 )
843 defaultFluxMag0T1 = pexConfig.Field(
844 dtype=float,
845 doc="Default value for fluxMag0T1 (for an unrecognized filter).",
846 default=pow(10.0, 0.4*28.0)
847 )
849 # Vignette correction configuration.
850 doVignette = pexConfig.Field(
851 dtype=bool,
852 doc="Apply vignetting parameters?",
853 default=False,
854 )
855 vignette = pexConfig.ConfigurableField(
856 target=VignetteTask,
857 doc="Vignetting task.",
858 )
860 # Transmission curve configuration.
861 doAttachTransmissionCurve = pexConfig.Field(
862 dtype=bool,
863 default=False,
864 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
865 )
866 doUseOpticsTransmission = pexConfig.Field(
867 dtype=bool,
868 default=True,
869 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
870 )
871 doUseFilterTransmission = pexConfig.Field(
872 dtype=bool,
873 default=True,
874 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
875 )
876 doUseSensorTransmission = pexConfig.Field(
877 dtype=bool,
878 default=True,
879 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
880 )
881 doUseAtmosphereTransmission = pexConfig.Field(
882 dtype=bool,
883 default=True,
884 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
885 )
887 # Illumination correction.
888 doIlluminationCorrection = pexConfig.Field(
889 dtype=bool,
890 default=False,
891 doc="Perform illumination correction?"
892 )
893 illuminationCorrectionDataProductName = pexConfig.Field(
894 dtype=str,
895 doc="Name of the illumination correction data product.",
896 default="illumcor",
897 )
898 illumScale = pexConfig.Field(
899 dtype=float,
900 doc="Scale factor for the illumination correction.",
901 default=1.0,
902 )
903 illumFilters = pexConfig.ListField(
904 dtype=str,
905 default=[],
906 doc="Only perform illumination correction for these filters."
907 )
909 # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
910 doWrite = pexConfig.Field(
911 dtype=bool,
912 doc="Persist postISRCCD?",
913 default=True,
914 )
916 def validate(self):
917 super().validate()
918 if self.doFlat and self.doApplyGains:
919 raise ValueError("You may not specify both doFlat and doApplyGains")
920 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib:
921 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
922 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate:
923 self.maskListToInterpolate.append(self.saturatedMaskName)
924 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate:
925 self.maskListToInterpolate.remove(self.saturatedMaskName)
926 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
927 self.maskListToInterpolate.append("UNMASKEDNAN")
930class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
931 """Apply common instrument signature correction algorithms to a raw frame.
933 The process for correcting imaging data is very similar from
934 camera to camera. This task provides a vanilla implementation of
935 doing these corrections, including the ability to turn certain
936 corrections off if they are not needed. The inputs to the primary
937 method, `run()`, are a raw exposure to be corrected and the
938 calibration data products. The raw input is a single chip sized
939 mosaic of all amps including overscans and other non-science
940 pixels. The method `runDataRef()` identifies and defines the
941 calibration data products, and is intended for use by a
942 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
943 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
944 subclassed for different camera, although the most camera specific
945 methods have been split into subtasks that can be redirected
946 appropriately.
948 The __init__ method sets up the subtasks for ISR processing, using
949 the defaults from `lsst.ip.isr`.
951 Parameters
952 ----------
953 args : `list`
954 Positional arguments passed to the Task constructor. None used at this time.
955 kwargs : `dict`, optional
956 Keyword arguments passed on to the Task constructor. None used at this time.
957 """
958 ConfigClass = IsrTaskConfig
959 _DefaultName = "isr"
961 def __init__(self, **kwargs):
962 super().__init__(**kwargs)
963 self.makeSubtask("assembleCcd")
964 self.makeSubtask("crosstalk")
965 self.makeSubtask("strayLight")
966 self.makeSubtask("fringe")
967 self.makeSubtask("masking")
968 self.makeSubtask("overscan")
969 self.makeSubtask("vignette")
971 def runQuantum(self, butlerQC, inputRefs, outputRefs):
972 inputs = butlerQC.get(inputRefs)
974 try:
975 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
976 except Exception as e:
977 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
978 (inputRefs, e))
980 inputs['isGen3'] = True
982 detector = inputs['ccdExposure'].getDetector()
984 if self.config.doCrosstalk is True:
985 # Crosstalk sources need to be defined by the pipeline
986 # yaml if they exist.
987 if 'crosstalk' in inputs and inputs['crosstalk'] is not None:
988 if not isinstance(inputs['crosstalk'], CrosstalkCalib):
989 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk'])
990 else:
991 coeffVector = (self.config.crosstalk.crosstalkValues
992 if self.config.crosstalk.useConfigCoefficients else None)
993 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
994 inputs['crosstalk'] = crosstalkCalib
995 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0:
996 if 'crosstalkSources' not in inputs:
997 self.log.warning("No crosstalkSources found for chip with interChip terms!")
999 if self.doLinearize(detector) is True:
1000 if 'linearizer' in inputs:
1001 if isinstance(inputs['linearizer'], dict):
1002 linearizer = linearize.Linearizer(detector=detector, log=self.log)
1003 linearizer.fromYaml(inputs['linearizer'])
1004 self.log.warning("Dictionary linearizers will be deprecated in DM-28741.")
1005 elif isinstance(inputs['linearizer'], numpy.ndarray):
1006 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None),
1007 detector=detector,
1008 log=self.log)
1009 self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.")
1010 else:
1011 linearizer = inputs['linearizer']
1012 linearizer.log = self.log
1013 inputs['linearizer'] = linearizer
1014 else:
1015 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log)
1016 self.log.warning("Constructing linearizer from cameraGeom information.")
1018 if self.config.doDefect is True:
1019 if "defects" in inputs and inputs['defects'] is not None:
1020 # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
1021 # masking expects a list of defects defined by their bounding box
1022 if not isinstance(inputs["defects"], Defects):
1023 inputs["defects"] = Defects.fromTable(inputs["defects"])
1025 # Load the correct style of brighter-fatter kernel, and repack
1026 # the information as a numpy array.
1027 if self.config.doBrighterFatter:
1028 brighterFatterKernel = inputs.pop('newBFKernel', None)
1029 if brighterFatterKernel is None:
1030 brighterFatterKernel = inputs.get('bfKernel', None)
1032 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1033 # This is a ISR calib kernel
1034 detName = detector.getName()
1035 level = brighterFatterKernel.level
1037 # This is expected to be a dictionary of amp-wise gains.
1038 inputs['bfGains'] = brighterFatterKernel.gain
1039 if self.config.brighterFatterLevel == 'DETECTOR':
1040 if level == 'DETECTOR':
1041 if detName in brighterFatterKernel.detKernels:
1042 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1043 else:
1044 raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1045 elif level == 'AMP':
1046 self.log.warning("Making DETECTOR level kernel from AMP based brighter "
1047 "fatter kernels.")
1048 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1049 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1050 elif self.config.brighterFatterLevel == 'AMP':
1051 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1053 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
1054 expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
1055 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
1056 expId=expId,
1057 assembler=self.assembleCcd
1058 if self.config.doAssembleIsrExposures else None)
1059 else:
1060 inputs['fringes'] = pipeBase.Struct(fringes=None)
1062 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
1063 if 'strayLightData' not in inputs:
1064 inputs['strayLightData'] = None
1066 outputs = self.run(**inputs)
1067 butlerQC.put(outputs, outputRefs)
1069 def readIsrData(self, dataRef, rawExposure):
1070 """Retrieve necessary frames for instrument signature removal.
1072 Pre-fetching all required ISR data products limits the IO
1073 required by the ISR. Any conflict between the calibration data
1074 available and that needed for ISR is also detected prior to
1075 doing processing, allowing it to fail quickly.
1077 Parameters
1078 ----------
1079 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1080 Butler reference of the detector data to be processed
1081 rawExposure : `afw.image.Exposure`
1082 The raw exposure that will later be corrected with the
1083 retrieved calibration data; should not be modified in this
1084 method.
1086 Returns
1087 -------
1088 result : `lsst.pipe.base.Struct`
1089 Result struct with components (which may be `None`):
1090 - ``bias``: bias calibration frame (`afw.image.Exposure`)
1091 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
1092 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1093 - ``dark``: dark calibration frame (`afw.image.Exposure`)
1094 - ``flat``: flat calibration frame (`afw.image.Exposure`)
1095 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1096 - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1097 - ``fringes``: `lsst.pipe.base.Struct` with components:
1098 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1099 - ``seed``: random seed derived from the ccdExposureId for random
1100 number generator (`uint32`).
1101 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1102 A ``TransmissionCurve`` that represents the throughput of the optics,
1103 to be evaluated in focal-plane coordinates.
1104 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1105 A ``TransmissionCurve`` that represents the throughput of the filter
1106 itself, to be evaluated in focal-plane coordinates.
1107 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1108 A ``TransmissionCurve`` that represents the throughput of the sensor
1109 itself, to be evaluated in post-assembly trimmed detector coordinates.
1110 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1111 A ``TransmissionCurve`` that represents the throughput of the
1112 atmosphere, assumed to be spatially constant.
1113 - ``strayLightData`` : `object`
1114 An opaque object containing calibration information for
1115 stray-light correction. If `None`, no correction will be
1116 performed.
1117 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1119 Raises
1120 ------
1121 NotImplementedError :
1122 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1123 """
1124 try:
1125 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1126 dateObs = dateObs.toPython().isoformat()
1127 except RuntimeError:
1128 self.log.warning("Unable to identify dateObs for rawExposure.")
1129 dateObs = None
1131 ccd = rawExposure.getDetector()
1132 filterLabel = rawExposure.getFilterLabel()
1133 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1134 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
1135 biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
1136 if self.config.doBias else None)
1137 # immediate=True required for functors and linearizers are functors; see ticket DM-6515
1138 linearizer = (dataRef.get("linearizer", immediate=True)
1139 if self.doLinearize(ccd) else None)
1140 if linearizer is not None and not isinstance(linearizer, numpy.ndarray):
1141 linearizer.log = self.log
1142 if isinstance(linearizer, numpy.ndarray):
1143 linearizer = linearize.Linearizer(table=linearizer, detector=ccd)
1145 crosstalkCalib = None
1146 if self.config.doCrosstalk:
1147 try:
1148 crosstalkCalib = dataRef.get("crosstalk", immediate=True)
1149 except NoResults:
1150 coeffVector = (self.config.crosstalk.crosstalkValues
1151 if self.config.crosstalk.useConfigCoefficients else None)
1152 crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1153 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1154 if self.config.doCrosstalk else None)
1156 darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
1157 if self.config.doDark else None)
1158 flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName,
1159 dateObs=dateObs)
1160 if self.config.doFlat else None)
1162 brighterFatterKernel = None
1163 brighterFatterGains = None
1164 if self.config.doBrighterFatter is True:
1165 try:
1166 # Use the new-style cp_pipe version of the kernel if it exists
1167 # If using a new-style kernel, always use the self-consistent
1168 # gains, i.e. the ones inside the kernel object itself
1169 brighterFatterKernel = dataRef.get("brighterFatterKernel")
1170 brighterFatterGains = brighterFatterKernel.gain
1171 self.log.info("New style brighter-fatter kernel (brighterFatterKernel) loaded")
1172 except NoResults:
1173 try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
1174 brighterFatterKernel = dataRef.get("bfKernel")
1175 self.log.info("Old style brighter-fatter kernel (bfKernel) loaded")
1176 except NoResults:
1177 brighterFatterKernel = None
1178 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1179 # If the kernel is not an ndarray, it's the cp_pipe version
1180 # so extract the kernel for this detector, or raise an error
1181 if self.config.brighterFatterLevel == 'DETECTOR':
1182 if brighterFatterKernel.detKernels:
1183 brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()]
1184 else:
1185 raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1186 else:
1187 # TODO DM-15631 for implementing this
1188 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1190 defectList = (dataRef.get("defects")
1191 if self.config.doDefect else None)
1192 expId = rawExposure.getInfo().getVisitInfo().getExposureId()
1193 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1194 if self.config.doAssembleIsrExposures else None)
1195 if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1196 else pipeBase.Struct(fringes=None))
1198 if self.config.doAttachTransmissionCurve:
1199 opticsTransmission = (dataRef.get("transmission_optics")
1200 if self.config.doUseOpticsTransmission else None)
1201 filterTransmission = (dataRef.get("transmission_filter")
1202 if self.config.doUseFilterTransmission else None)
1203 sensorTransmission = (dataRef.get("transmission_sensor")
1204 if self.config.doUseSensorTransmission else None)
1205 atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1206 if self.config.doUseAtmosphereTransmission else None)
1207 else:
1208 opticsTransmission = None
1209 filterTransmission = None
1210 sensorTransmission = None
1211 atmosphereTransmission = None
1213 if self.config.doStrayLight:
1214 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1215 else:
1216 strayLightData = None
1218 illumMaskedImage = (self.getIsrExposure(dataRef,
1219 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1220 if (self.config.doIlluminationCorrection
1221 and physicalFilter in self.config.illumFilters)
1222 else None)
1224 # Struct should include only kwargs to run()
1225 return pipeBase.Struct(bias=biasExposure,
1226 linearizer=linearizer,
1227 crosstalk=crosstalkCalib,
1228 crosstalkSources=crosstalkSources,
1229 dark=darkExposure,
1230 flat=flatExposure,
1231 bfKernel=brighterFatterKernel,
1232 bfGains=brighterFatterGains,
1233 defects=defectList,
1234 fringes=fringeStruct,
1235 opticsTransmission=opticsTransmission,
1236 filterTransmission=filterTransmission,
1237 sensorTransmission=sensorTransmission,
1238 atmosphereTransmission=atmosphereTransmission,
1239 strayLightData=strayLightData,
1240 illumMaskedImage=illumMaskedImage
1241 )
1243 @pipeBase.timeMethod
1244 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1245 crosstalk=None, crosstalkSources=None,
1246 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1247 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1248 sensorTransmission=None, atmosphereTransmission=None,
1249 detectorNum=None, strayLightData=None, illumMaskedImage=None,
1250 isGen3=False,
1251 ):
1252 """Perform instrument signature removal on an exposure.
1254 Steps included in the ISR processing, in order performed, are:
1255 - saturation and suspect pixel masking
1256 - overscan subtraction
1257 - CCD assembly of individual amplifiers
1258 - bias subtraction
1259 - variance image construction
1260 - linearization of non-linear response
1261 - crosstalk masking
1262 - brighter-fatter correction
1263 - dark subtraction
1264 - fringe correction
1265 - stray light subtraction
1266 - flat correction
1267 - masking of known defects and camera specific features
1268 - vignette calculation
1269 - appending transmission curve and distortion model
1271 Parameters
1272 ----------
1273 ccdExposure : `lsst.afw.image.Exposure`
1274 The raw exposure that is to be run through ISR. The
1275 exposure is modified by this method.
1276 camera : `lsst.afw.cameraGeom.Camera`, optional
1277 The camera geometry for this exposure. Required if ``isGen3`` is
1278 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1279 ``flat`` does not have an associated detector.
1280 bias : `lsst.afw.image.Exposure`, optional
1281 Bias calibration frame.
1282 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1283 Functor for linearization.
1284 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1285 Calibration for crosstalk.
1286 crosstalkSources : `list`, optional
1287 List of possible crosstalk sources.
1288 dark : `lsst.afw.image.Exposure`, optional
1289 Dark calibration frame.
1290 flat : `lsst.afw.image.Exposure`, optional
1291 Flat calibration frame.
1292 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
1293 Photon transfer curve dataset, with, e.g., gains
1294 and read noise.
1295 bfKernel : `numpy.ndarray`, optional
1296 Brighter-fatter kernel.
1297 bfGains : `dict` of `float`, optional
1298 Gains used to override the detector's nominal gains for the
1299 brighter-fatter correction. A dict keyed by amplifier name for
1300 the detector in question.
1301 defects : `lsst.ip.isr.Defects`, optional
1302 List of defects.
1303 fringes : `lsst.pipe.base.Struct`, optional
1304 Struct containing the fringe correction data, with
1305 elements:
1306 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1307 - ``seed``: random seed derived from the ccdExposureId for random
1308 number generator (`uint32`)
1309 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1310 A ``TransmissionCurve`` that represents the throughput of the optics,
1311 to be evaluated in focal-plane coordinates.
1312 filterTransmission : `lsst.afw.image.TransmissionCurve`
1313 A ``TransmissionCurve`` that represents the throughput of the filter
1314 itself, to be evaluated in focal-plane coordinates.
1315 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1316 A ``TransmissionCurve`` that represents the throughput of the sensor
1317 itself, to be evaluated in post-assembly trimmed detector coordinates.
1318 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1319 A ``TransmissionCurve`` that represents the throughput of the
1320 atmosphere, assumed to be spatially constant.
1321 detectorNum : `int`, optional
1322 The integer number for the detector to process.
1323 isGen3 : bool, optional
1324 Flag this call to run() as using the Gen3 butler environment.
1325 strayLightData : `object`, optional
1326 Opaque object containing calibration information for stray-light
1327 correction. If `None`, no correction will be performed.
1328 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1329 Illumination correction image.
1331 Returns
1332 -------
1333 result : `lsst.pipe.base.Struct`
1334 Result struct with component:
1335 - ``exposure`` : `afw.image.Exposure`
1336 The fully ISR corrected exposure.
1337 - ``outputExposure`` : `afw.image.Exposure`
1338 An alias for `exposure`
1339 - ``ossThumb`` : `numpy.ndarray`
1340 Thumbnail image of the exposure after overscan subtraction.
1341 - ``flattenedThumb`` : `numpy.ndarray`
1342 Thumbnail image of the exposure after flat-field correction.
1344 Raises
1345 ------
1346 RuntimeError
1347 Raised if a configuration option is set to True, but the
1348 required calibration data has not been specified.
1350 Notes
1351 -----
1352 The current processed exposure can be viewed by setting the
1353 appropriate lsstDebug entries in the `debug.display`
1354 dictionary. The names of these entries correspond to some of
1355 the IsrTaskConfig Boolean options, with the value denoting the
1356 frame to use. The exposure is shown inside the matching
1357 option check and after the processing of that step has
1358 finished. The steps with debug points are:
1360 doAssembleCcd
1361 doBias
1362 doCrosstalk
1363 doBrighterFatter
1364 doDark
1365 doFringe
1366 doStrayLight
1367 doFlat
1369 In addition, setting the "postISRCCD" entry displays the
1370 exposure after all ISR processing has finished.
1372 """
1374 if isGen3 is True:
1375 # Gen3 currently cannot automatically do configuration overrides.
1376 # DM-15257 looks to discuss this issue.
1377 # Configure input exposures;
1378 if detectorNum is None:
1379 raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1381 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1382 bias = self.ensureExposure(bias, camera, detectorNum)
1383 dark = self.ensureExposure(dark, camera, detectorNum)
1384 flat = self.ensureExposure(flat, camera, detectorNum)
1385 else:
1386 if isinstance(ccdExposure, ButlerDataRef):
1387 return self.runDataRef(ccdExposure)
1389 ccd = ccdExposure.getDetector()
1390 filterLabel = ccdExposure.getFilterLabel()
1391 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1393 if not ccd:
1394 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1395 ccd = [FakeAmp(ccdExposure, self.config)]
1397 # Validate Input
1398 if self.config.doBias and bias is None:
1399 raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1400 if self.doLinearize(ccd) and linearizer is None:
1401 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1402 if self.config.doBrighterFatter and bfKernel is None:
1403 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1404 if self.config.doDark and dark is None:
1405 raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1406 if self.config.doFlat and flat is None:
1407 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1408 if self.config.doDefect and defects is None:
1409 raise RuntimeError("Must supply defects if config.doDefect=True.")
1410 if (self.config.doFringe and physicalFilter in self.fringe.config.filters
1411 and fringes.fringes is None):
1412 # The `fringes` object needs to be a pipeBase.Struct, as
1413 # we use it as a `dict` for the parameters of
1414 # `FringeTask.run()`. The `fringes.fringes` `list` may
1415 # not be `None` if `doFringe=True`. Otherwise, raise.
1416 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1417 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters
1418 and illumMaskedImage is None):
1419 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1421 # Begin ISR processing.
1422 if self.config.doConvertIntToFloat:
1423 self.log.info("Converting exposure to floating point values.")
1424 ccdExposure = self.convertIntToFloat(ccdExposure)
1426 if self.config.doBias and self.config.doBiasBeforeOverscan:
1427 self.log.info("Applying bias correction.")
1428 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1429 trimToFit=self.config.doTrimToMatchCalib)
1430 self.debugView(ccdExposure, "doBias")
1432 # Amplifier level processing.
1433 overscans = []
1434 for amp in ccd:
1435 # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1436 if ccdExposure.getBBox().contains(amp.getBBox()):
1437 # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1438 badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1440 if self.config.doOverscan and not badAmp:
1441 # Overscan correction on amp-by-amp basis.
1442 overscanResults = self.overscanCorrection(ccdExposure, amp)
1443 self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1444 if overscanResults is not None and \
1445 self.config.qa is not None and self.config.qa.saveStats is True:
1446 if isinstance(overscanResults.overscanFit, float):
1447 qaMedian = overscanResults.overscanFit
1448 qaStdev = float("NaN")
1449 else:
1450 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1451 afwMath.MEDIAN | afwMath.STDEVCLIP)
1452 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1453 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1455 self.metadata.set(f"FIT MEDIAN {amp.getName()}", qaMedian)
1456 self.metadata.set(f"FIT STDEV {amp.getName()}", qaStdev)
1457 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1458 amp.getName(), qaMedian, qaStdev)
1460 # Residuals after overscan correction
1461 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1462 afwMath.MEDIAN | afwMath.STDEVCLIP)
1463 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1464 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1466 self.metadata.set(f"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter)
1467 self.metadata.set(f"RESIDUAL STDEV {amp.getName()}", qaStdevAfter)
1468 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f",
1469 amp.getName(), qaMedianAfter, qaStdevAfter)
1471 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1472 else:
1473 if badAmp:
1474 self.log.warning("Amplifier %s is bad.", amp.getName())
1475 overscanResults = None
1477 overscans.append(overscanResults if overscanResults is not None else None)
1478 else:
1479 self.log.info("Skipped OSCAN for %s.", amp.getName())
1481 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1482 self.log.info("Applying crosstalk correction.")
1483 self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1484 crosstalkSources=crosstalkSources, camera=camera)
1485 self.debugView(ccdExposure, "doCrosstalk")
1487 if self.config.doAssembleCcd:
1488 self.log.info("Assembling CCD from amplifiers.")
1489 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1491 if self.config.expectWcs and not ccdExposure.getWcs():
1492 self.log.warning("No WCS found in input exposure.")
1493 self.debugView(ccdExposure, "doAssembleCcd")
1495 ossThumb = None
1496 if self.config.qa.doThumbnailOss:
1497 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1499 if self.config.doBias and not self.config.doBiasBeforeOverscan:
1500 self.log.info("Applying bias correction.")
1501 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1502 trimToFit=self.config.doTrimToMatchCalib)
1503 self.debugView(ccdExposure, "doBias")
1505 if self.config.doVariance:
1506 for amp, overscanResults in zip(ccd, overscans):
1507 if ccdExposure.getBBox().contains(amp.getBBox()):
1508 self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1509 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1510 if overscanResults is not None:
1511 self.updateVariance(ampExposure, amp,
1512 overscanImage=overscanResults.overscanImage,
1513 ptcDataset=ptc)
1514 else:
1515 self.updateVariance(ampExposure, amp,
1516 overscanImage=None,
1517 ptcDataset=ptc)
1518 if self.config.qa is not None and self.config.qa.saveStats is True:
1519 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1520 afwMath.MEDIAN | afwMath.STDEVCLIP)
1521 self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1522 qaStats.getValue(afwMath.MEDIAN))
1523 self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1524 qaStats.getValue(afwMath.STDEVCLIP))
1525 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1526 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1527 qaStats.getValue(afwMath.STDEVCLIP))
1528 if self.config.maskNegativeVariance:
1529 self.maskNegativeVariance(ccdExposure)
1531 if self.doLinearize(ccd):
1532 self.log.info("Applying linearizer.")
1533 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1534 detector=ccd, log=self.log)
1536 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1537 self.log.info("Applying crosstalk correction.")
1538 self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1539 crosstalkSources=crosstalkSources, isTrimmed=True)
1540 self.debugView(ccdExposure, "doCrosstalk")
1542 # Masking block. Optionally mask known defects, NAN/inf pixels, widen trails, and do
1543 # anything else the camera needs. Saturated and suspect pixels have already been masked.
1544 if self.config.doDefect:
1545 self.log.info("Masking defects.")
1546 self.maskDefect(ccdExposure, defects)
1548 if self.config.numEdgeSuspect > 0:
1549 self.log.info("Masking edges as SUSPECT.")
1550 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1551 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
1553 if self.config.doNanMasking:
1554 self.log.info("Masking non-finite (NAN, inf) value pixels.")
1555 self.maskNan(ccdExposure)
1557 if self.config.doWidenSaturationTrails:
1558 self.log.info("Widening saturation trails.")
1559 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1561 if self.config.doCameraSpecificMasking:
1562 self.log.info("Masking regions for camera specific reasons.")
1563 self.masking.run(ccdExposure)
1565 if self.config.doBrighterFatter:
1566 # We need to apply flats and darks before we can interpolate, and we
1567 # need to interpolate before we do B-F, but we do B-F without the
1568 # flats and darks applied so we can work in units of electrons or holes.
1569 # This context manager applies and then removes the darks and flats.
1570 #
1571 # We also do not want to interpolate values here, so operate on temporary
1572 # images so we can apply only the BF-correction and roll back the
1573 # interpolation.
1574 interpExp = ccdExposure.clone()
1575 with self.flatContext(interpExp, flat, dark):
1576 isrFunctions.interpolateFromMask(
1577 maskedImage=interpExp.getMaskedImage(),
1578 fwhm=self.config.fwhm,
1579 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1580 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1581 )
1582 bfExp = interpExp.clone()
1584 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.",
1585 type(bfKernel), type(bfGains))
1586 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1587 self.config.brighterFatterMaxIter,
1588 self.config.brighterFatterThreshold,
1589 self.config.brighterFatterApplyGain,
1590 bfGains)
1591 if bfResults[1] == self.config.brighterFatterMaxIter:
1592 self.log.warning("Brighter-fatter correction did not converge, final difference %f.",
1593 bfResults[0])
1594 else:
1595 self.log.info("Finished brighter-fatter correction in %d iterations.",
1596 bfResults[1])
1597 image = ccdExposure.getMaskedImage().getImage()
1598 bfCorr = bfExp.getMaskedImage().getImage()
1599 bfCorr -= interpExp.getMaskedImage().getImage()
1600 image += bfCorr
1602 # Applying the brighter-fatter correction applies a
1603 # convolution to the science image. At the edges this
1604 # convolution may not have sufficient valid pixels to
1605 # produce a valid correction. Mark pixels within the size
1606 # of the brighter-fatter kernel as EDGE to warn of this
1607 # fact.
1608 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1609 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1610 maskPlane="EDGE")
1612 if self.config.brighterFatterMaskGrowSize > 0:
1613 self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1614 for maskPlane in self.config.brighterFatterMaskListToInterpolate:
1615 isrFunctions.growMasks(ccdExposure.getMask(),
1616 radius=self.config.brighterFatterMaskGrowSize,
1617 maskNameList=maskPlane,
1618 maskValue=maskPlane)
1620 self.debugView(ccdExposure, "doBrighterFatter")
1622 if self.config.doDark:
1623 self.log.info("Applying dark correction.")
1624 self.darkCorrection(ccdExposure, dark)
1625 self.debugView(ccdExposure, "doDark")
1627 if self.config.doFringe and not self.config.fringeAfterFlat:
1628 self.log.info("Applying fringe correction before flat.")
1629 self.fringe.run(ccdExposure, **fringes.getDict())
1630 self.debugView(ccdExposure, "doFringe")
1632 if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1633 self.log.info("Checking strayLight correction.")
1634 self.strayLight.run(ccdExposure, strayLightData)
1635 self.debugView(ccdExposure, "doStrayLight")
1637 if self.config.doFlat:
1638 self.log.info("Applying flat correction.")
1639 self.flatCorrection(ccdExposure, flat)
1640 self.debugView(ccdExposure, "doFlat")
1642 if self.config.doApplyGains:
1643 self.log.info("Applying gain correction instead of flat.")
1644 if self.config.usePtcGains:
1645 self.log.info("Using gains from the Photon Transfer Curve.")
1646 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1647 ptcGains=ptc.gain)
1648 else:
1649 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1651 if self.config.doFringe and self.config.fringeAfterFlat:
1652 self.log.info("Applying fringe correction after flat.")
1653 self.fringe.run(ccdExposure, **fringes.getDict())
1655 if self.config.doVignette:
1656 self.log.info("Constructing Vignette polygon.")
1657 self.vignettePolygon = self.vignette.run(ccdExposure)
1659 if self.config.vignette.doWriteVignettePolygon:
1660 self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1662 if self.config.doAttachTransmissionCurve:
1663 self.log.info("Adding transmission curves.")
1664 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1665 filterTransmission=filterTransmission,
1666 sensorTransmission=sensorTransmission,
1667 atmosphereTransmission=atmosphereTransmission)
1669 flattenedThumb = None
1670 if self.config.qa.doThumbnailFlattened:
1671 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1673 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters:
1674 self.log.info("Performing illumination correction.")
1675 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1676 illumMaskedImage, illumScale=self.config.illumScale,
1677 trimToFit=self.config.doTrimToMatchCalib)
1679 preInterpExp = None
1680 if self.config.doSaveInterpPixels:
1681 preInterpExp = ccdExposure.clone()
1683 # Reset and interpolate bad pixels.
1684 #
1685 # Large contiguous bad regions (which should have the BAD mask
1686 # bit set) should have their values set to the image median.
1687 # This group should include defects and bad amplifiers. As the
1688 # area covered by these defects are large, there's little
1689 # reason to expect that interpolation would provide a more
1690 # useful value.
1691 #
1692 # Smaller defects can be safely interpolated after the larger
1693 # regions have had their pixel values reset. This ensures
1694 # that the remaining defects adjacent to bad amplifiers (as an
1695 # example) do not attempt to interpolate extreme values.
1696 if self.config.doSetBadRegions:
1697 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1698 if badPixelCount > 0:
1699 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1701 if self.config.doInterpolate:
1702 self.log.info("Interpolating masked pixels.")
1703 isrFunctions.interpolateFromMask(
1704 maskedImage=ccdExposure.getMaskedImage(),
1705 fwhm=self.config.fwhm,
1706 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1707 maskNameList=list(self.config.maskListToInterpolate)
1708 )
1710 self.roughZeroPoint(ccdExposure)
1712 if self.config.doMeasureBackground:
1713 self.log.info("Measuring background level.")
1714 self.measureBackground(ccdExposure, self.config.qa)
1716 if self.config.qa is not None and self.config.qa.saveStats is True:
1717 for amp in ccd:
1718 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1719 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1720 afwMath.MEDIAN | afwMath.STDEVCLIP)
1721 self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1722 qaStats.getValue(afwMath.MEDIAN))
1723 self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1724 qaStats.getValue(afwMath.STDEVCLIP))
1725 self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1726 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1727 qaStats.getValue(afwMath.STDEVCLIP))
1729 self.debugView(ccdExposure, "postISRCCD")
1731 return pipeBase.Struct(
1732 exposure=ccdExposure,
1733 ossThumb=ossThumb,
1734 flattenedThumb=flattenedThumb,
1736 preInterpExposure=preInterpExp,
1737 outputExposure=ccdExposure,
1738 outputOssThumbnail=ossThumb,
1739 outputFlattenedThumbnail=flattenedThumb,
1740 )
1742 @pipeBase.timeMethod
1743 def runDataRef(self, sensorRef):
1744 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1746 This method contains the `CmdLineTask` interface to the ISR
1747 processing. All IO is handled here, freeing the `run()` method
1748 to manage only pixel-level calculations. The steps performed
1749 are:
1750 - Read in necessary detrending/isr/calibration data.
1751 - Process raw exposure in `run()`.
1752 - Persist the ISR-corrected exposure as "postISRCCD" if
1753 config.doWrite=True.
1755 Parameters
1756 ----------
1757 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1758 DataRef of the detector data to be processed
1760 Returns
1761 -------
1762 result : `lsst.pipe.base.Struct`
1763 Result struct with component:
1764 - ``exposure`` : `afw.image.Exposure`
1765 The fully ISR corrected exposure.
1767 Raises
1768 ------
1769 RuntimeError
1770 Raised if a configuration option is set to True, but the
1771 required calibration data does not exist.
1773 """
1774 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1776 ccdExposure = sensorRef.get(self.config.datasetType)
1778 camera = sensorRef.get("camera")
1779 isrData = self.readIsrData(sensorRef, ccdExposure)
1781 result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1783 if self.config.doWrite:
1784 sensorRef.put(result.exposure, "postISRCCD")
1785 if result.preInterpExposure is not None:
1786 sensorRef.put(result.preInterpExposure, "postISRCCD_uninterpolated")
1787 if result.ossThumb is not None:
1788 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1789 if result.flattenedThumb is not None:
1790 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1792 return result
1794 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1795 """Retrieve a calibration dataset for removing instrument signature.
1797 Parameters
1798 ----------
1800 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1801 DataRef of the detector data to find calibration datasets
1802 for.
1803 datasetType : `str`
1804 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1805 dateObs : `str`, optional
1806 Date of the observation. Used to correct butler failures
1807 when using fallback filters.
1808 immediate : `Bool`
1809 If True, disable butler proxies to enable error handling
1810 within this routine.
1812 Returns
1813 -------
1814 exposure : `lsst.afw.image.Exposure`
1815 Requested calibration frame.
1817 Raises
1818 ------
1819 RuntimeError
1820 Raised if no matching calibration frame can be found.
1821 """
1822 try:
1823 exp = dataRef.get(datasetType, immediate=immediate)
1824 except Exception as exc1:
1825 if not self.config.fallbackFilterName:
1826 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1827 try:
1828 if self.config.useFallbackDate and dateObs:
1829 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1830 dateObs=dateObs, immediate=immediate)
1831 else:
1832 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1833 except Exception as exc2:
1834 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1835 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1836 self.log.warning("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1838 if self.config.doAssembleIsrExposures:
1839 exp = self.assembleCcd.assembleCcd(exp)
1840 return exp
1842 def ensureExposure(self, inputExp, camera, detectorNum):
1843 """Ensure that the data returned by Butler is a fully constructed exposure.
1845 ISR requires exposure-level image data for historical reasons, so if we did
1846 not recieve that from Butler, construct it from what we have, modifying the
1847 input in place.
1849 Parameters
1850 ----------
1851 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1852 `lsst.afw.image.ImageF`
1853 The input data structure obtained from Butler.
1854 camera : `lsst.afw.cameraGeom.camera`
1855 The camera associated with the image. Used to find the appropriate
1856 detector.
1857 detectorNum : `int`
1858 The detector this exposure should match.
1860 Returns
1861 -------
1862 inputExp : `lsst.afw.image.Exposure`
1863 The re-constructed exposure, with appropriate detector parameters.
1865 Raises
1866 ------
1867 TypeError
1868 Raised if the input data cannot be used to construct an exposure.
1869 """
1870 if isinstance(inputExp, afwImage.DecoratedImageU):
1871 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1872 elif isinstance(inputExp, afwImage.ImageF):
1873 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1874 elif isinstance(inputExp, afwImage.MaskedImageF):
1875 inputExp = afwImage.makeExposure(inputExp)
1876 elif isinstance(inputExp, afwImage.Exposure):
1877 pass
1878 elif inputExp is None:
1879 # Assume this will be caught by the setup if it is a problem.
1880 return inputExp
1881 else:
1882 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1883 (type(inputExp), ))
1885 if inputExp.getDetector() is None:
1886 inputExp.setDetector(camera[detectorNum])
1888 return inputExp
1890 def convertIntToFloat(self, exposure):
1891 """Convert exposure image from uint16 to float.
1893 If the exposure does not need to be converted, the input is
1894 immediately returned. For exposures that are converted to use
1895 floating point pixels, the variance is set to unity and the
1896 mask to zero.
1898 Parameters
1899 ----------
1900 exposure : `lsst.afw.image.Exposure`
1901 The raw exposure to be converted.
1903 Returns
1904 -------
1905 newexposure : `lsst.afw.image.Exposure`
1906 The input ``exposure``, converted to floating point pixels.
1908 Raises
1909 ------
1910 RuntimeError
1911 Raised if the exposure type cannot be converted to float.
1913 """
1914 if isinstance(exposure, afwImage.ExposureF):
1915 # Nothing to be done
1916 self.log.debug("Exposure already of type float.")
1917 return exposure
1918 if not hasattr(exposure, "convertF"):
1919 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1921 newexposure = exposure.convertF()
1922 newexposure.variance[:] = 1
1923 newexposure.mask[:] = 0x0
1925 return newexposure
1927 def maskAmplifier(self, ccdExposure, amp, defects):
1928 """Identify bad amplifiers, saturated and suspect pixels.
1930 Parameters
1931 ----------
1932 ccdExposure : `lsst.afw.image.Exposure`
1933 Input exposure to be masked.
1934 amp : `lsst.afw.table.AmpInfoCatalog`
1935 Catalog of parameters defining the amplifier on this
1936 exposure to mask.
1937 defects : `lsst.ip.isr.Defects`
1938 List of defects. Used to determine if the entire
1939 amplifier is bad.
1941 Returns
1942 -------
1943 badAmp : `Bool`
1944 If this is true, the entire amplifier area is covered by
1945 defects and unusable.
1947 """
1948 maskedImage = ccdExposure.getMaskedImage()
1950 badAmp = False
1952 # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1953 # comparison with current defects definition.
1954 if defects is not None:
1955 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1957 # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1958 # association with pixels in current ccdExposure).
1959 if badAmp:
1960 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1961 afwImage.PARENT)
1962 maskView = dataView.getMask()
1963 maskView |= maskView.getPlaneBitMask("BAD")
1964 del maskView
1965 return badAmp
1967 # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1968 # Saturation and suspect pixels can be masked now, though.
1969 limits = dict()
1970 if self.config.doSaturation and not badAmp:
1971 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1972 if self.config.doSuspect and not badAmp:
1973 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1974 if math.isfinite(self.config.saturation):
1975 limits.update({self.config.saturatedMaskName: self.config.saturation})
1977 for maskName, maskThreshold in limits.items():
1978 if not math.isnan(maskThreshold):
1979 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1980 isrFunctions.makeThresholdMask(
1981 maskedImage=dataView,
1982 threshold=maskThreshold,
1983 growFootprints=0,
1984 maskName=maskName
1985 )
1987 # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1988 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1989 afwImage.PARENT)
1990 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1991 self.config.suspectMaskName])
1992 if numpy.all(maskView.getArray() & maskVal > 0):
1993 badAmp = True
1994 maskView |= maskView.getPlaneBitMask("BAD")
1996 return badAmp
1998 def overscanCorrection(self, ccdExposure, amp):
1999 """Apply overscan correction in place.
2001 This method does initial pixel rejection of the overscan
2002 region. The overscan can also be optionally segmented to
2003 allow for discontinuous overscan responses to be fit
2004 separately. The actual overscan subtraction is performed by
2005 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
2006 which is called here after the amplifier is preprocessed.
2008 Parameters
2009 ----------
2010 ccdExposure : `lsst.afw.image.Exposure`
2011 Exposure to have overscan correction performed.
2012 amp : `lsst.afw.cameraGeom.Amplifer`
2013 The amplifier to consider while correcting the overscan.
2015 Returns
2016 -------
2017 overscanResults : `lsst.pipe.base.Struct`
2018 Result struct with components:
2019 - ``imageFit`` : scalar or `lsst.afw.image.Image`
2020 Value or fit subtracted from the amplifier image data.
2021 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
2022 Value or fit subtracted from the overscan image data.
2023 - ``overscanImage`` : `lsst.afw.image.Image`
2024 Image of the overscan region with the overscan
2025 correction applied. This quantity is used to estimate
2026 the amplifier read noise empirically.
2028 Raises
2029 ------
2030 RuntimeError
2031 Raised if the ``amp`` does not contain raw pixel information.
2033 See Also
2034 --------
2035 lsst.ip.isr.isrFunctions.overscanCorrection
2036 """
2037 if amp.getRawHorizontalOverscanBBox().isEmpty():
2038 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
2039 return None
2041 statControl = afwMath.StatisticsControl()
2042 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2044 # Determine the bounding boxes
2045 dataBBox = amp.getRawDataBBox()
2046 oscanBBox = amp.getRawHorizontalOverscanBBox()
2047 dx0 = 0
2048 dx1 = 0
2050 prescanBBox = amp.getRawPrescanBBox()
2051 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
2052 dx0 += self.config.overscanNumLeadingColumnsToSkip
2053 dx1 -= self.config.overscanNumTrailingColumnsToSkip
2054 else:
2055 dx0 += self.config.overscanNumTrailingColumnsToSkip
2056 dx1 -= self.config.overscanNumLeadingColumnsToSkip
2058 # Determine if we need to work on subregions of the amplifier and overscan.
2059 imageBBoxes = []
2060 overscanBBoxes = []
2062 if ((self.config.overscanBiasJump
2063 and self.config.overscanBiasJumpLocation)
2064 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2065 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
2066 self.config.overscanBiasJumpDevices)):
2067 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
2068 yLower = self.config.overscanBiasJumpLocation
2069 yUpper = dataBBox.getHeight() - yLower
2070 else:
2071 yUpper = self.config.overscanBiasJumpLocation
2072 yLower = dataBBox.getHeight() - yUpper
2074 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2075 lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
2076 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2077 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2078 yLower)))
2080 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
2081 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
2082 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
2083 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2084 yUpper)))
2085 else:
2086 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2087 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
2088 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2089 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2090 oscanBBox.getHeight())))
2092 # Perform overscan correction on subregions, ensuring saturated pixels are masked.
2093 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
2094 ampImage = ccdExposure.maskedImage[imageBBox]
2095 overscanImage = ccdExposure.maskedImage[overscanBBox]
2097 overscanArray = overscanImage.image.array
2098 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2099 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2100 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
2102 statControl = afwMath.StatisticsControl()
2103 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2105 overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp)
2107 # Measure average overscan levels and record them in the metadata.
2108 levelStat = afwMath.MEDIAN
2109 sigmaStat = afwMath.STDEVCLIP
2111 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2112 self.config.qa.flatness.nIter)
2113 metadata = ccdExposure.getMetadata()
2114 ampNum = amp.getName()
2115 # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
2116 if isinstance(overscanResults.overscanFit, float):
2117 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2118 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2119 else:
2120 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2121 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2122 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2124 return overscanResults
2126 def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None):
2127 """Set the variance plane using the gain and read noise
2129 The read noise is calculated from the ``overscanImage`` if the
2130 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2131 the value from the amplifier data is used.
2133 Parameters
2134 ----------
2135 ampExposure : `lsst.afw.image.Exposure`
2136 Exposure to process.
2137 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2138 Amplifier detector data.
2139 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2140 Image of overscan, required only for empirical read noise.
2141 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
2142 PTC dataset containing the gains and read noise.
2145 Raises
2146 ------
2147 RuntimeError
2148 Raised if either ``usePtcGains`` of ``usePtcReadNoise``
2149 are ``True``, but ptcDataset is not provided.
2151 Raised if ```doEmpiricalReadNoise`` is ``True`` but
2152 ``overscanImage`` is ``None``.
2154 See also
2155 --------
2156 lsst.ip.isr.isrFunctions.updateVariance
2157 """
2158 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2159 if self.config.usePtcGains:
2160 if ptcDataset is None:
2161 raise RuntimeError("No ptcDataset provided to use PTC gains.")
2162 else:
2163 gain = ptcDataset.gain[amp.getName()]
2164 self.log.info("Using gain from Photon Transfer Curve.")
2165 else:
2166 gain = amp.getGain()
2168 if math.isnan(gain):
2169 gain = 1.0
2170 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2171 elif gain <= 0:
2172 patchedGain = 1.0
2173 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.",
2174 amp.getName(), gain, patchedGain)
2175 gain = patchedGain
2177 if self.config.doEmpiricalReadNoise and overscanImage is None:
2178 raise RuntimeError("Overscan is none for EmpiricalReadNoise.")
2180 if self.config.doEmpiricalReadNoise and overscanImage is not None:
2181 stats = afwMath.StatisticsControl()
2182 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2183 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2184 self.log.info("Calculated empirical read noise for amp %s: %f.",
2185 amp.getName(), readNoise)
2186 elif self.config.usePtcReadNoise:
2187 if ptcDataset is None:
2188 raise RuntimeError("No ptcDataset provided to use PTC readnoise.")
2189 else:
2190 readNoise = ptcDataset.noise[amp.getName()]
2191 self.log.info("Using read noise from Photon Transfer Curve.")
2192 else:
2193 readNoise = amp.getReadNoise()
2195 isrFunctions.updateVariance(
2196 maskedImage=ampExposure.getMaskedImage(),
2197 gain=gain,
2198 readNoise=readNoise,
2199 )
2201 def maskNegativeVariance(self, exposure):
2202 """Identify and mask pixels with negative variance values.
2204 Parameters
2205 ----------
2206 exposure : `lsst.afw.image.Exposure`
2207 Exposure to process.
2209 See Also
2210 --------
2211 lsst.ip.isr.isrFunctions.updateVariance
2212 """
2213 maskPlane = exposure.getMask().getPlaneBitMask(self.config.negativeVarianceMaskName)
2214 bad = numpy.where(exposure.getVariance().getArray() <= 0.0)
2215 exposure.mask.array[bad] |= maskPlane
2217 def darkCorrection(self, exposure, darkExposure, invert=False):
2218 """Apply dark correction in place.
2220 Parameters
2221 ----------
2222 exposure : `lsst.afw.image.Exposure`
2223 Exposure to process.
2224 darkExposure : `lsst.afw.image.Exposure`
2225 Dark exposure of the same size as ``exposure``.
2226 invert : `Bool`, optional
2227 If True, re-add the dark to an already corrected image.
2229 Raises
2230 ------
2231 RuntimeError
2232 Raised if either ``exposure`` or ``darkExposure`` do not
2233 have their dark time defined.
2235 See Also
2236 --------
2237 lsst.ip.isr.isrFunctions.darkCorrection
2238 """
2239 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2240 if math.isnan(expScale):
2241 raise RuntimeError("Exposure darktime is NAN.")
2242 if darkExposure.getInfo().getVisitInfo() is not None \
2243 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2244 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2245 else:
2246 # DM-17444: darkExposure.getInfo.getVisitInfo() is None
2247 # so getDarkTime() does not exist.
2248 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2249 darkScale = 1.0
2251 isrFunctions.darkCorrection(
2252 maskedImage=exposure.getMaskedImage(),
2253 darkMaskedImage=darkExposure.getMaskedImage(),
2254 expScale=expScale,
2255 darkScale=darkScale,
2256 invert=invert,
2257 trimToFit=self.config.doTrimToMatchCalib
2258 )
2260 def doLinearize(self, detector):
2261 """Check if linearization is needed for the detector cameraGeom.
2263 Checks config.doLinearize and the linearity type of the first
2264 amplifier.
2266 Parameters
2267 ----------
2268 detector : `lsst.afw.cameraGeom.Detector`
2269 Detector to get linearity type from.
2271 Returns
2272 -------
2273 doLinearize : `Bool`
2274 If True, linearization should be performed.
2275 """
2276 return self.config.doLinearize and \
2277 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2279 def flatCorrection(self, exposure, flatExposure, invert=False):
2280 """Apply flat correction in place.
2282 Parameters
2283 ----------
2284 exposure : `lsst.afw.image.Exposure`
2285 Exposure to process.
2286 flatExposure : `lsst.afw.image.Exposure`
2287 Flat exposure of the same size as ``exposure``.
2288 invert : `Bool`, optional
2289 If True, unflatten an already flattened image.
2291 See Also
2292 --------
2293 lsst.ip.isr.isrFunctions.flatCorrection
2294 """
2295 isrFunctions.flatCorrection(
2296 maskedImage=exposure.getMaskedImage(),
2297 flatMaskedImage=flatExposure.getMaskedImage(),
2298 scalingType=self.config.flatScalingType,
2299 userScale=self.config.flatUserScale,
2300 invert=invert,
2301 trimToFit=self.config.doTrimToMatchCalib
2302 )
2304 def saturationDetection(self, exposure, amp):
2305 """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2307 Parameters
2308 ----------
2309 exposure : `lsst.afw.image.Exposure`
2310 Exposure to process. Only the amplifier DataSec is processed.
2311 amp : `lsst.afw.table.AmpInfoCatalog`
2312 Amplifier detector data.
2314 See Also
2315 --------
2316 lsst.ip.isr.isrFunctions.makeThresholdMask
2317 """
2318 if not math.isnan(amp.getSaturation()):
2319 maskedImage = exposure.getMaskedImage()
2320 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2321 isrFunctions.makeThresholdMask(
2322 maskedImage=dataView,
2323 threshold=amp.getSaturation(),
2324 growFootprints=0,
2325 maskName=self.config.saturatedMaskName,
2326 )
2328 def saturationInterpolation(self, exposure):
2329 """Interpolate over saturated pixels, in place.
2331 This method should be called after `saturationDetection`, to
2332 ensure that the saturated pixels have been identified in the
2333 SAT mask. It should also be called after `assembleCcd`, since
2334 saturated regions may cross amplifier boundaries.
2336 Parameters
2337 ----------
2338 exposure : `lsst.afw.image.Exposure`
2339 Exposure to process.
2341 See Also
2342 --------
2343 lsst.ip.isr.isrTask.saturationDetection
2344 lsst.ip.isr.isrFunctions.interpolateFromMask
2345 """
2346 isrFunctions.interpolateFromMask(
2347 maskedImage=exposure.getMaskedImage(),
2348 fwhm=self.config.fwhm,
2349 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2350 maskNameList=list(self.config.saturatedMaskName),
2351 )
2353 def suspectDetection(self, exposure, amp):
2354 """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2356 Parameters
2357 ----------
2358 exposure : `lsst.afw.image.Exposure`
2359 Exposure to process. Only the amplifier DataSec is processed.
2360 amp : `lsst.afw.table.AmpInfoCatalog`
2361 Amplifier detector data.
2363 See Also
2364 --------
2365 lsst.ip.isr.isrFunctions.makeThresholdMask
2367 Notes
2368 -----
2369 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2370 This is intended to indicate pixels that may be affected by unknown systematics;
2371 for example if non-linearity corrections above a certain level are unstable
2372 then that would be a useful value for suspectLevel. A value of `nan` indicates
2373 that no such level exists and no pixels are to be masked as suspicious.
2374 """
2375 suspectLevel = amp.getSuspectLevel()
2376 if math.isnan(suspectLevel):
2377 return
2379 maskedImage = exposure.getMaskedImage()
2380 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2381 isrFunctions.makeThresholdMask(
2382 maskedImage=dataView,
2383 threshold=suspectLevel,
2384 growFootprints=0,
2385 maskName=self.config.suspectMaskName,
2386 )
2388 def maskDefect(self, exposure, defectBaseList):
2389 """Mask defects using mask plane "BAD", in place.
2391 Parameters
2392 ----------
2393 exposure : `lsst.afw.image.Exposure`
2394 Exposure to process.
2395 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2396 `lsst.afw.image.DefectBase`.
2397 List of defects to mask.
2399 Notes
2400 -----
2401 Call this after CCD assembly, since defects may cross amplifier boundaries.
2402 """
2403 maskedImage = exposure.getMaskedImage()
2404 if not isinstance(defectBaseList, Defects):
2405 # Promotes DefectBase to Defect
2406 defectList = Defects(defectBaseList)
2407 else:
2408 defectList = defectBaseList
2409 defectList.maskPixels(maskedImage, maskName="BAD")
2411 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2412 """Mask edge pixels with applicable mask plane.
2414 Parameters
2415 ----------
2416 exposure : `lsst.afw.image.Exposure`
2417 Exposure to process.
2418 numEdgePixels : `int`, optional
2419 Number of edge pixels to mask.
2420 maskPlane : `str`, optional
2421 Mask plane name to use.
2422 level : `str`, optional
2423 Level at which to mask edges.
2424 """
2425 maskedImage = exposure.getMaskedImage()
2426 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2428 if numEdgePixels > 0:
2429 if level == 'DETECTOR':
2430 boxes = [maskedImage.getBBox()]
2431 elif level == 'AMP':
2432 boxes = [amp.getBBox() for amp in exposure.getDetector()]
2434 for box in boxes:
2435 # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2436 subImage = maskedImage[box]
2437 box.grow(-numEdgePixels)
2438 # Mask pixels outside box
2439 SourceDetectionTask.setEdgeBits(
2440 subImage,
2441 box,
2442 maskBitMask)
2444 def maskAndInterpolateDefects(self, exposure, defectBaseList):
2445 """Mask and interpolate defects using mask plane "BAD", in place.
2447 Parameters
2448 ----------
2449 exposure : `lsst.afw.image.Exposure`
2450 Exposure to process.
2451 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2452 `lsst.afw.image.DefectBase`.
2453 List of defects to mask and interpolate.
2455 See Also
2456 --------
2457 lsst.ip.isr.isrTask.maskDefect
2458 """
2459 self.maskDefect(exposure, defectBaseList)
2460 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2461 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2462 isrFunctions.interpolateFromMask(
2463 maskedImage=exposure.getMaskedImage(),
2464 fwhm=self.config.fwhm,
2465 growSaturatedFootprints=0,
2466 maskNameList=["BAD"],
2467 )
2469 def maskNan(self, exposure):
2470 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2472 Parameters
2473 ----------
2474 exposure : `lsst.afw.image.Exposure`
2475 Exposure to process.
2477 Notes
2478 -----
2479 We mask over all non-finite values (NaN, inf), including those
2480 that are masked with other bits (because those may or may not be
2481 interpolated over later, and we want to remove all NaN/infs).
2482 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2483 preserve the historical name.
2484 """
2485 maskedImage = exposure.getMaskedImage()
2487 # Find and mask NaNs
2488 maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2489 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2490 numNans = maskNans(maskedImage, maskVal)
2491 self.metadata.set("NUMNANS", numNans)
2492 if numNans > 0:
2493 self.log.warning("There were %d unmasked NaNs.", numNans)
2495 def maskAndInterpolateNan(self, exposure):
2496 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2497 in place.
2499 Parameters
2500 ----------
2501 exposure : `lsst.afw.image.Exposure`
2502 Exposure to process.
2504 See Also
2505 --------
2506 lsst.ip.isr.isrTask.maskNan
2507 """
2508 self.maskNan(exposure)
2509 isrFunctions.interpolateFromMask(
2510 maskedImage=exposure.getMaskedImage(),
2511 fwhm=self.config.fwhm,
2512 growSaturatedFootprints=0,
2513 maskNameList=["UNMASKEDNAN"],
2514 )
2516 def measureBackground(self, exposure, IsrQaConfig=None):
2517 """Measure the image background in subgrids, for quality control purposes.
2519 Parameters
2520 ----------
2521 exposure : `lsst.afw.image.Exposure`
2522 Exposure to process.
2523 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2524 Configuration object containing parameters on which background
2525 statistics and subgrids to use.
2526 """
2527 if IsrQaConfig is not None:
2528 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2529 IsrQaConfig.flatness.nIter)
2530 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2531 statsControl.setAndMask(maskVal)
2532 maskedImage = exposure.getMaskedImage()
2533 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2534 skyLevel = stats.getValue(afwMath.MEDIAN)
2535 skySigma = stats.getValue(afwMath.STDEVCLIP)
2536 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2537 metadata = exposure.getMetadata()
2538 metadata.set('SKYLEVEL', skyLevel)
2539 metadata.set('SKYSIGMA', skySigma)
2541 # calcluating flatlevel over the subgrids
2542 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2543 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2544 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2545 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2546 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2547 skyLevels = numpy.zeros((nX, nY))
2549 for j in range(nY):
2550 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2551 for i in range(nX):
2552 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2554 xLLC = xc - meshXHalf
2555 yLLC = yc - meshYHalf
2556 xURC = xc + meshXHalf - 1
2557 yURC = yc + meshYHalf - 1
2559 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2560 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2562 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2564 good = numpy.where(numpy.isfinite(skyLevels))
2565 skyMedian = numpy.median(skyLevels[good])
2566 flatness = (skyLevels[good] - skyMedian) / skyMedian
2567 flatness_rms = numpy.std(flatness)
2568 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2570 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2571 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2572 nX, nY, flatness_pp, flatness_rms)
2574 metadata.set('FLATNESS_PP', float(flatness_pp))
2575 metadata.set('FLATNESS_RMS', float(flatness_rms))
2576 metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2577 metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2578 metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2580 def roughZeroPoint(self, exposure):
2581 """Set an approximate magnitude zero point for the exposure.
2583 Parameters
2584 ----------
2585 exposure : `lsst.afw.image.Exposure`
2586 Exposure to process.
2587 """
2588 filterLabel = exposure.getFilterLabel()
2589 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2591 if physicalFilter in self.config.fluxMag0T1:
2592 fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2593 else:
2594 self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter)
2595 fluxMag0 = self.config.defaultFluxMag0T1
2597 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2598 if not expTime > 0: # handle NaN as well as <= 0
2599 self.log.warning("Non-positive exposure time; skipping rough zero point.")
2600 return
2602 self.log.info("Setting rough magnitude zero point for filter %s: %f",
2603 physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2604 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2606 def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2607 """Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2609 Parameters
2610 ----------
2611 ccdExposure : `lsst.afw.image.Exposure`
2612 Exposure to process.
2613 fpPolygon : `lsst.afw.geom.Polygon`
2614 Polygon in focal plane coordinates.
2615 """
2616 # Get ccd corners in focal plane coordinates
2617 ccd = ccdExposure.getDetector()
2618 fpCorners = ccd.getCorners(FOCAL_PLANE)
2619 ccdPolygon = Polygon(fpCorners)
2621 # Get intersection of ccd corners with fpPolygon
2622 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2624 # Transform back to pixel positions and build new polygon
2625 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2626 validPolygon = Polygon(ccdPoints)
2627 ccdExposure.getInfo().setValidPolygon(validPolygon)
2629 @contextmanager
2630 def flatContext(self, exp, flat, dark=None):
2631 """Context manager that applies and removes flats and darks,
2632 if the task is configured to apply them.
2634 Parameters
2635 ----------
2636 exp : `lsst.afw.image.Exposure`
2637 Exposure to process.
2638 flat : `lsst.afw.image.Exposure`
2639 Flat exposure the same size as ``exp``.
2640 dark : `lsst.afw.image.Exposure`, optional
2641 Dark exposure the same size as ``exp``.
2643 Yields
2644 ------
2645 exp : `lsst.afw.image.Exposure`
2646 The flat and dark corrected exposure.
2647 """
2648 if self.config.doDark and dark is not None:
2649 self.darkCorrection(exp, dark)
2650 if self.config.doFlat:
2651 self.flatCorrection(exp, flat)
2652 try:
2653 yield exp
2654 finally:
2655 if self.config.doFlat:
2656 self.flatCorrection(exp, flat, invert=True)
2657 if self.config.doDark and dark is not None:
2658 self.darkCorrection(exp, dark, invert=True)
2660 def debugView(self, exposure, stepname):
2661 """Utility function to examine ISR exposure at different stages.
2663 Parameters
2664 ----------
2665 exposure : `lsst.afw.image.Exposure`
2666 Exposure to view.
2667 stepname : `str`
2668 State of processing to view.
2669 """
2670 frame = getDebugFrame(self._display, stepname)
2671 if frame:
2672 display = getDisplay(frame)
2673 display.scale('asinh', 'zscale')
2674 display.mtv(exposure)
2675 prompt = "Press Enter to continue [c]... "
2676 while True:
2677 ans = input(prompt).lower()
2678 if ans in ("", "c",):
2679 break
2682class FakeAmp(object):
2683 """A Detector-like object that supports returning gain and saturation level
2685 This is used when the input exposure does not have a detector.
2687 Parameters
2688 ----------
2689 exposure : `lsst.afw.image.Exposure`
2690 Exposure to generate a fake amplifier for.
2691 config : `lsst.ip.isr.isrTaskConfig`
2692 Configuration to apply to the fake amplifier.
2693 """
2695 def __init__(self, exposure, config):
2696 self._bbox = exposure.getBBox(afwImage.LOCAL)
2697 self._RawHorizontalOverscanBBox = lsst.geom.Box2I()
2698 self._gain = config.gain
2699 self._readNoise = config.readNoise
2700 self._saturation = config.saturation
2702 def getBBox(self):
2703 return self._bbox
2705 def getRawBBox(self):
2706 return self._bbox
2708 def getRawHorizontalOverscanBBox(self):
2709 return self._RawHorizontalOverscanBBox
2711 def getGain(self):
2712 return self._gain
2714 def getReadNoise(self):
2715 return self._readNoise
2717 def getSaturation(self):
2718 return self._saturation
2720 def getSuspectLevel(self):
2721 return float("NaN")
2724class RunIsrConfig(pexConfig.Config):
2725 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2728class RunIsrTask(pipeBase.CmdLineTask):
2729 """Task to wrap the default IsrTask to allow it to be retargeted.
2731 The standard IsrTask can be called directly from a command line
2732 program, but doing so removes the ability of the task to be
2733 retargeted. As most cameras override some set of the IsrTask
2734 methods, this would remove those data-specific methods in the
2735 output post-ISR images. This wrapping class fixes the issue,
2736 allowing identical post-ISR images to be generated by both the
2737 processCcd and isrTask code.
2738 """
2739 ConfigClass = RunIsrConfig
2740 _DefaultName = "runIsr"
2742 def __init__(self, *args, **kwargs):
2743 super().__init__(*args, **kwargs)
2744 self.makeSubtask("isr")
2746 def runDataRef(self, dataRef):
2747 """
2748 Parameters
2749 ----------
2750 dataRef : `lsst.daf.persistence.ButlerDataRef`
2751 data reference of the detector data to be processed
2753 Returns
2754 -------
2755 result : `pipeBase.Struct`
2756 Result struct with component:
2758 - exposure : `lsst.afw.image.Exposure`
2759 Post-ISR processed exposure.
2760 """
2761 return self.isr.runDataRef(dataRef)