Coverage for python/lsst/ip/isr/isrTask.py : 17%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import math
23import numpy
25import lsst.geom
26import lsst.afw.image as afwImage
27import lsst.afw.math as afwMath
28import lsst.pex.config as pexConfig
29import lsst.pipe.base as pipeBase
30import lsst.pipe.base.connectionTypes as cT
32from contextlib import contextmanager
33from lsstDebug import getDebugFrame
35from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36 ReadoutCorner)
37from lsst.afw.display import getDisplay
38from lsst.afw.geom import Polygon
39from lsst.daf.persistence import ButlerDataRef
40from lsst.daf.persistence.butler import NoResults
41from lsst.meas.algorithms.detection import SourceDetectionTask
42from lsst.meas.algorithms import Defects
44from . import isrFunctions
45from . import isrQa
46from . import linearize
48from .assembleCcdTask import AssembleCcdTask
49from .crosstalk import CrosstalkTask, CrosstalkCalib
50from .fringe import FringeTask
51from .isr import maskNans
52from .masking import MaskingTask
53from .overscan import OverscanCorrectionTask
54from .straylight import StrayLightTask
55from .vignette import VignetteTask
56from lsst.daf.butler import DataCoordinate, DimensionGraph, DimensionUniverse
59__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
62def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections):
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
67 populated.
69 This will be unused until DM-25348 resolves the quantum graph
70 generation issue.
72 Parameters
73 ----------
74 datasetType : `str`
75 Dataset to lookup.
76 registry : `lsst.daf.butler.Registry`
77 Butler registry to query.
78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79 Data id to transform to identify crosstalkSources. The
80 ``detector`` entry will be stripped.
81 collections : `lsst.daf.butler.CollectionSearch`
82 Collections to search through.
84 Returns
85 -------
86 results : `list` [`lsst.afw.image.Exposure`]
87 List of datasets that match the query that will be used as
88 crosstalkSources.
89 """
90 newDataId = DataCoordinate(DimensionGraph(DimensionUniverse(),
91 names=('instrument', 'exposure')),
92 (quantumDataId['instrument'], quantumDataId['exposure']))
93 results = list(registry.queryDatasets(datasetType,
94 collections=collections,
95 dataId=newDataId,
96 deduplicate=True,
97 expand=True))
98 return(results)
101class IsrTaskConnections(pipeBase.PipelineTaskConnections,
102 dimensions={"instrument", "exposure", "detector"},
103 defaultTemplates={}):
104 ccdExposure = cT.Input(
105 name="raw",
106 doc="Input exposure to process.",
107 storageClass="Exposure",
108 dimensions=["instrument", "exposure", "detector"],
109 )
110 camera = cT.PrerequisiteInput(
111 name="camera",
112 storageClass="Camera",
113 doc="Input camera to construct complete exposures.",
114 dimensions=["instrument", "calibration_label"],
115 )
117 crosstalk = cT.PrerequisiteInput(
118 name="crosstalk",
119 doc="Input crosstalk object",
120 storageClass="CrosstalkCalib",
121 dimensions=["instrument", "calibration_label", "detector"],
122 )
123 # TODO: DM-25348. This does not work yet to correctly load
124 # possible crosstalk sources.
125 crosstalkSources = cT.PrerequisiteInput(
126 name="isrOverscanCorrected",
127 doc="Overscan corrected input images.",
128 storageClass="Exposure",
129 dimensions=["instrument", "exposure", "detector"],
130 deferLoad=True,
131 multiple=True,
132 lookupFunction=crosstalkSourceLookup,
133 )
134 bias = cT.PrerequisiteInput(
135 name="bias",
136 doc="Input bias calibration.",
137 storageClass="ExposureF",
138 dimensions=["instrument", "calibration_label", "detector"],
139 )
140 dark = cT.PrerequisiteInput(
141 name='dark',
142 doc="Input dark calibration.",
143 storageClass="ExposureF",
144 dimensions=["instrument", "calibration_label", "detector"],
145 )
146 flat = cT.PrerequisiteInput(
147 name="flat",
148 doc="Input flat calibration.",
149 storageClass="ExposureF",
150 dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
151 )
152 fringes = cT.PrerequisiteInput(
153 name="fringe",
154 doc="Input fringe calibration.",
155 storageClass="ExposureF",
156 dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
157 )
158 strayLightData = cT.PrerequisiteInput(
159 name='yBackground',
160 doc="Input stray light calibration.",
161 storageClass="StrayLightData",
162 dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
163 )
164 bfKernel = cT.PrerequisiteInput(
165 name='bfKernel',
166 doc="Input brighter-fatter kernel.",
167 storageClass="NumpyArray",
168 dimensions=["instrument", "calibration_label"],
169 )
170 newBFKernel = cT.PrerequisiteInput(
171 name='brighterFatterKernel',
172 doc="Newer complete kernel + gain solutions.",
173 storageClass="BrighterFatterKernel",
174 dimensions=["instrument", "calibration_label", "detector"],
175 )
176 defects = cT.PrerequisiteInput(
177 name='defects',
178 doc="Input defect tables.",
179 storageClass="Defects",
180 dimensions=["instrument", "calibration_label", "detector"],
181 )
182 opticsTransmission = cT.PrerequisiteInput(
183 name="transmission_optics",
184 storageClass="TransmissionCurve",
185 doc="Transmission curve due to the optics.",
186 dimensions=["instrument", "calibration_label"],
187 )
188 filterTransmission = cT.PrerequisiteInput(
189 name="transmission_filter",
190 storageClass="TransmissionCurve",
191 doc="Transmission curve due to the filter.",
192 dimensions=["instrument", "physical_filter", "calibration_label"],
193 )
194 sensorTransmission = cT.PrerequisiteInput(
195 name="transmission_sensor",
196 storageClass="TransmissionCurve",
197 doc="Transmission curve due to the sensor.",
198 dimensions=["instrument", "calibration_label", "detector"],
199 )
200 atmosphereTransmission = cT.PrerequisiteInput(
201 name="transmission_atmosphere",
202 storageClass="TransmissionCurve",
203 doc="Transmission curve due to the atmosphere.",
204 dimensions=["instrument"],
205 )
206 illumMaskedImage = cT.PrerequisiteInput(
207 name="illum",
208 doc="Input illumination correction.",
209 storageClass="MaskedImageF",
210 dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
211 )
213 outputExposure = cT.Output(
214 name='postISRCCD',
215 doc="Output ISR processed exposure.",
216 storageClass="Exposure",
217 dimensions=["instrument", "exposure", "detector"],
218 )
219 preInterpExposure = cT.Output(
220 name='preInterpISRCCD',
221 doc="Output ISR processed exposure, with pixels left uninterpolated.",
222 storageClass="ExposureF",
223 dimensions=["instrument", "exposure", "detector"],
224 )
225 outputOssThumbnail = cT.Output(
226 name="OssThumb",
227 doc="Output Overscan-subtracted thumbnail image.",
228 storageClass="Thumbnail",
229 dimensions=["instrument", "exposure", "detector"],
230 )
231 outputFlattenedThumbnail = cT.Output(
232 name="FlattenedThumb",
233 doc="Output flat-corrected thumbnail image.",
234 storageClass="Thumbnail",
235 dimensions=["instrument", "exposure", "detector"],
236 )
238 def __init__(self, *, config=None):
239 super().__init__(config=config)
241 if config.doBias is not True:
242 self.prerequisiteInputs.discard("bias")
243 if config.doLinearize is not True:
244 self.prerequisiteInputs.discard("linearizer")
245 if config.doCrosstalk is not True:
246 self.inputs.discard("crosstalkSources")
247 self.prerequisiteInputs.discard("crosstalk")
248 if config.doBrighterFatter is not True:
249 self.prerequisiteInputs.discard("bfKernel")
250 self.prerequisiteInputs.discard("newBFKernel")
251 if config.doDefect is not True:
252 self.prerequisiteInputs.discard("defects")
253 if config.doDark is not True:
254 self.prerequisiteInputs.discard("dark")
255 if config.doFlat is not True:
256 self.prerequisiteInputs.discard("flat")
257 if config.doAttachTransmissionCurve is not True:
258 self.prerequisiteInputs.discard("opticsTransmission")
259 self.prerequisiteInputs.discard("filterTransmission")
260 self.prerequisiteInputs.discard("sensorTransmission")
261 self.prerequisiteInputs.discard("atmosphereTransmission")
262 if config.doUseOpticsTransmission is not True:
263 self.prerequisiteInputs.discard("opticsTransmission")
264 if config.doUseFilterTransmission is not True:
265 self.prerequisiteInputs.discard("filterTransmission")
266 if config.doUseSensorTransmission is not True:
267 self.prerequisiteInputs.discard("sensorTransmission")
268 if config.doUseAtmosphereTransmission is not True:
269 self.prerequisiteInputs.discard("atmosphereTransmission")
270 if config.doIlluminationCorrection is not True:
271 self.prerequisiteInputs.discard("illumMaskedImage")
273 if config.doWrite is not True:
274 self.outputs.discard("outputExposure")
275 self.outputs.discard("preInterpExposure")
276 self.outputs.discard("outputFlattenedThumbnail")
277 self.outputs.discard("outputOssThumbnail")
278 if config.doSaveInterpPixels is not True:
279 self.outputs.discard("preInterpExposure")
280 if config.qa.doThumbnailOss is not True:
281 self.outputs.discard("outputOssThumbnail")
282 if config.qa.doThumbnailFlattened is not True:
283 self.outputs.discard("outputFlattenedThumbnail")
286class IsrTaskConfig(pipeBase.PipelineTaskConfig,
287 pipelineConnections=IsrTaskConnections):
288 """Configuration parameters for IsrTask.
290 Items are grouped in the order in which they are executed by the task.
291 """
292 datasetType = pexConfig.Field(
293 dtype=str,
294 doc="Dataset type for input data; users will typically leave this alone, "
295 "but camera-specific ISR tasks will override it",
296 default="raw",
297 )
299 fallbackFilterName = pexConfig.Field(
300 dtype=str,
301 doc="Fallback default filter name for calibrations.",
302 optional=True
303 )
304 useFallbackDate = pexConfig.Field(
305 dtype=bool,
306 doc="Pass observation date when using fallback filter.",
307 default=False,
308 )
309 expectWcs = pexConfig.Field(
310 dtype=bool,
311 default=True,
312 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
313 )
314 fwhm = pexConfig.Field(
315 dtype=float,
316 doc="FWHM of PSF in arcseconds.",
317 default=1.0,
318 )
319 qa = pexConfig.ConfigField(
320 dtype=isrQa.IsrQaConfig,
321 doc="QA related configuration options.",
322 )
324 # Image conversion configuration
325 doConvertIntToFloat = pexConfig.Field(
326 dtype=bool,
327 doc="Convert integer raw images to floating point values?",
328 default=True,
329 )
331 # Saturated pixel handling.
332 doSaturation = pexConfig.Field(
333 dtype=bool,
334 doc="Mask saturated pixels? NB: this is totally independent of the"
335 " interpolation option - this is ONLY setting the bits in the mask."
336 " To have them interpolated make sure doSaturationInterpolation=True",
337 default=True,
338 )
339 saturatedMaskName = pexConfig.Field(
340 dtype=str,
341 doc="Name of mask plane to use in saturation detection and interpolation",
342 default="SAT",
343 )
344 saturation = pexConfig.Field(
345 dtype=float,
346 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
347 default=float("NaN"),
348 )
349 growSaturationFootprintSize = pexConfig.Field(
350 dtype=int,
351 doc="Number of pixels by which to grow the saturation footprints",
352 default=1,
353 )
355 # Suspect pixel handling.
356 doSuspect = pexConfig.Field(
357 dtype=bool,
358 doc="Mask suspect pixels?",
359 default=False,
360 )
361 suspectMaskName = pexConfig.Field(
362 dtype=str,
363 doc="Name of mask plane to use for suspect pixels",
364 default="SUSPECT",
365 )
366 numEdgeSuspect = pexConfig.Field(
367 dtype=int,
368 doc="Number of edge pixels to be flagged as untrustworthy.",
369 default=0,
370 )
372 # Initial masking options.
373 doSetBadRegions = pexConfig.Field(
374 dtype=bool,
375 doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
376 default=True,
377 )
378 badStatistic = pexConfig.ChoiceField(
379 dtype=str,
380 doc="How to estimate the average value for BAD regions.",
381 default='MEANCLIP',
382 allowed={
383 "MEANCLIP": "Correct using the (clipped) mean of good data",
384 "MEDIAN": "Correct using the median of the good data",
385 },
386 )
388 # Overscan subtraction configuration.
389 doOverscan = pexConfig.Field(
390 dtype=bool,
391 doc="Do overscan subtraction?",
392 default=True,
393 )
394 overscan = pexConfig.ConfigurableField(
395 target=OverscanCorrectionTask,
396 doc="Overscan subtraction task for image segments.",
397 )
399 overscanFitType = pexConfig.ChoiceField(
400 dtype=str,
401 doc="The method for fitting the overscan bias level.",
402 default='MEDIAN',
403 allowed={
404 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
405 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
406 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
407 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
408 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
409 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
410 "MEAN": "Correct using the mean of the overscan region",
411 "MEANCLIP": "Correct using a clipped mean of the overscan region",
412 "MEDIAN": "Correct using the median of the overscan region",
413 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
414 },
415 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." +
416 " This option will no longer be used, and will be removed after v20.")
417 )
418 overscanOrder = pexConfig.Field(
419 dtype=int,
420 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, " +
421 "or number of spline knots if overscan fit type is a spline."),
422 default=1,
423 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." +
424 " This option will no longer be used, and will be removed after v20.")
425 )
426 overscanNumSigmaClip = pexConfig.Field(
427 dtype=float,
428 doc="Rejection threshold (sigma) for collapsing overscan before fit",
429 default=3.0,
430 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." +
431 " This option will no longer be used, and will be removed after v20.")
432 )
433 overscanIsInt = pexConfig.Field(
434 dtype=bool,
435 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN" +
436 " and overscan.FitType=MEDIAN_PER_ROW.",
437 default=True,
438 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface." +
439 " This option will no longer be used, and will be removed after v20.")
440 )
441 # These options do not get deprecated, as they define how we slice up the image data.
442 overscanNumLeadingColumnsToSkip = pexConfig.Field(
443 dtype=int,
444 doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
445 default=0,
446 )
447 overscanNumTrailingColumnsToSkip = pexConfig.Field(
448 dtype=int,
449 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
450 default=0,
451 )
452 overscanMaxDev = pexConfig.Field( 452 ↛ exitline 452 didn't jump to the function exit
453 dtype=float,
454 doc="Maximum deviation from the median for overscan",
455 default=1000.0, check=lambda x: x > 0
456 )
457 overscanBiasJump = pexConfig.Field(
458 dtype=bool,
459 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
460 default=False,
461 )
462 overscanBiasJumpKeyword = pexConfig.Field(
463 dtype=str,
464 doc="Header keyword containing information about devices.",
465 default="NO_SUCH_KEY",
466 )
467 overscanBiasJumpDevices = pexConfig.ListField(
468 dtype=str,
469 doc="List of devices that need piecewise overscan correction.",
470 default=(),
471 )
472 overscanBiasJumpLocation = pexConfig.Field(
473 dtype=int,
474 doc="Location of bias jump along y-axis.",
475 default=0,
476 )
478 # Amplifier to CCD assembly configuration
479 doAssembleCcd = pexConfig.Field(
480 dtype=bool,
481 default=True,
482 doc="Assemble amp-level exposures into a ccd-level exposure?"
483 )
484 assembleCcd = pexConfig.ConfigurableField(
485 target=AssembleCcdTask,
486 doc="CCD assembly task",
487 )
489 # General calibration configuration.
490 doAssembleIsrExposures = pexConfig.Field(
491 dtype=bool,
492 default=False,
493 doc="Assemble amp-level calibration exposures into ccd-level exposure?"
494 )
495 doTrimToMatchCalib = pexConfig.Field(
496 dtype=bool,
497 default=False,
498 doc="Trim raw data to match calibration bounding boxes?"
499 )
501 # Bias subtraction.
502 doBias = pexConfig.Field(
503 dtype=bool,
504 doc="Apply bias frame correction?",
505 default=True,
506 )
507 biasDataProductName = pexConfig.Field(
508 dtype=str,
509 doc="Name of the bias data product",
510 default="bias",
511 )
513 # Variance construction
514 doVariance = pexConfig.Field(
515 dtype=bool,
516 doc="Calculate variance?",
517 default=True
518 )
519 gain = pexConfig.Field(
520 dtype=float,
521 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
522 default=float("NaN"),
523 )
524 readNoise = pexConfig.Field(
525 dtype=float,
526 doc="The read noise to use if no Detector is present in the Exposure",
527 default=0.0,
528 )
529 doEmpiricalReadNoise = pexConfig.Field(
530 dtype=bool,
531 default=False,
532 doc="Calculate empirical read noise instead of value from AmpInfo data?"
533 )
535 # Linearization.
536 doLinearize = pexConfig.Field(
537 dtype=bool,
538 doc="Correct for nonlinearity of the detector's response?",
539 default=True,
540 )
542 # Crosstalk.
543 doCrosstalk = pexConfig.Field(
544 dtype=bool,
545 doc="Apply intra-CCD crosstalk correction?",
546 default=False,
547 )
548 doCrosstalkBeforeAssemble = pexConfig.Field(
549 dtype=bool,
550 doc="Apply crosstalk correction before CCD assembly, and before trimming?",
551 default=False,
552 )
553 crosstalk = pexConfig.ConfigurableField(
554 target=CrosstalkTask,
555 doc="Intra-CCD crosstalk correction",
556 )
558 # Masking options.
559 doDefect = pexConfig.Field(
560 dtype=bool,
561 doc="Apply correction for CCD defects, e.g. hot pixels?",
562 default=True,
563 )
564 doNanMasking = pexConfig.Field(
565 dtype=bool,
566 doc="Mask NAN pixels?",
567 default=True,
568 )
569 doWidenSaturationTrails = pexConfig.Field(
570 dtype=bool,
571 doc="Widen bleed trails based on their width?",
572 default=True
573 )
575 # Brighter-Fatter correction.
576 doBrighterFatter = pexConfig.Field(
577 dtype=bool,
578 default=False,
579 doc="Apply the brighter fatter correction"
580 )
581 brighterFatterLevel = pexConfig.ChoiceField(
582 dtype=str,
583 default="DETECTOR",
584 doc="The level at which to correct for brighter-fatter.",
585 allowed={
586 "AMP": "Every amplifier treated separately.",
587 "DETECTOR": "One kernel per detector",
588 }
589 )
590 brighterFatterMaxIter = pexConfig.Field(
591 dtype=int,
592 default=10,
593 doc="Maximum number of iterations for the brighter fatter correction"
594 )
595 brighterFatterThreshold = pexConfig.Field(
596 dtype=float,
597 default=1000,
598 doc="Threshold used to stop iterating the brighter fatter correction. It is the "
599 " absolute value of the difference between the current corrected image and the one"
600 " from the previous iteration summed over all the pixels."
601 )
602 brighterFatterApplyGain = pexConfig.Field(
603 dtype=bool,
604 default=True,
605 doc="Should the gain be applied when applying the brighter fatter correction?"
606 )
607 brighterFatterMaskGrowSize = pexConfig.Field(
608 dtype=int,
609 default=0,
610 doc="Number of pixels to grow the masks listed in config.maskListToInterpolate "
611 " when brighter-fatter correction is applied."
612 )
614 # Dark subtraction.
615 doDark = pexConfig.Field(
616 dtype=bool,
617 doc="Apply dark frame correction?",
618 default=True,
619 )
620 darkDataProductName = pexConfig.Field(
621 dtype=str,
622 doc="Name of the dark data product",
623 default="dark",
624 )
626 # Camera-specific stray light removal.
627 doStrayLight = pexConfig.Field(
628 dtype=bool,
629 doc="Subtract stray light in the y-band (due to encoder LEDs)?",
630 default=False,
631 )
632 strayLight = pexConfig.ConfigurableField(
633 target=StrayLightTask,
634 doc="y-band stray light correction"
635 )
637 # Flat correction.
638 doFlat = pexConfig.Field(
639 dtype=bool,
640 doc="Apply flat field correction?",
641 default=True,
642 )
643 flatDataProductName = pexConfig.Field(
644 dtype=str,
645 doc="Name of the flat data product",
646 default="flat",
647 )
648 flatScalingType = pexConfig.ChoiceField(
649 dtype=str,
650 doc="The method for scaling the flat on the fly.",
651 default='USER',
652 allowed={
653 "USER": "Scale by flatUserScale",
654 "MEAN": "Scale by the inverse of the mean",
655 "MEDIAN": "Scale by the inverse of the median",
656 },
657 )
658 flatUserScale = pexConfig.Field(
659 dtype=float,
660 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
661 default=1.0,
662 )
663 doTweakFlat = pexConfig.Field(
664 dtype=bool,
665 doc="Tweak flats to match observed amplifier ratios?",
666 default=False
667 )
669 # Amplifier normalization based on gains instead of using flats configuration.
670 doApplyGains = pexConfig.Field(
671 dtype=bool,
672 doc="Correct the amplifiers for their gains instead of applying flat correction",
673 default=False,
674 )
675 normalizeGains = pexConfig.Field(
676 dtype=bool,
677 doc="Normalize all the amplifiers in each CCD to have the same median value.",
678 default=False,
679 )
681 # Fringe correction.
682 doFringe = pexConfig.Field(
683 dtype=bool,
684 doc="Apply fringe correction?",
685 default=True,
686 )
687 fringe = pexConfig.ConfigurableField(
688 target=FringeTask,
689 doc="Fringe subtraction task",
690 )
691 fringeAfterFlat = pexConfig.Field(
692 dtype=bool,
693 doc="Do fringe subtraction after flat-fielding?",
694 default=True,
695 )
697 # Initial CCD-level background statistics options.
698 doMeasureBackground = pexConfig.Field(
699 dtype=bool,
700 doc="Measure the background level on the reduced image?",
701 default=False,
702 )
704 # Camera-specific masking configuration.
705 doCameraSpecificMasking = pexConfig.Field(
706 dtype=bool,
707 doc="Mask camera-specific bad regions?",
708 default=False,
709 )
710 masking = pexConfig.ConfigurableField(
711 target=MaskingTask,
712 doc="Masking task."
713 )
715 # Interpolation options.
717 doInterpolate = pexConfig.Field(
718 dtype=bool,
719 doc="Interpolate masked pixels?",
720 default=True,
721 )
722 doSaturationInterpolation = pexConfig.Field(
723 dtype=bool,
724 doc="Perform interpolation over pixels masked as saturated?"
725 " NB: This is independent of doSaturation; if that is False this plane"
726 " will likely be blank, resulting in a no-op here.",
727 default=True,
728 )
729 doNanInterpolation = pexConfig.Field(
730 dtype=bool,
731 doc="Perform interpolation over pixels masked as NaN?"
732 " NB: This is independent of doNanMasking; if that is False this plane"
733 " will likely be blank, resulting in a no-op here.",
734 default=True,
735 )
736 doNanInterpAfterFlat = pexConfig.Field(
737 dtype=bool,
738 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
739 "also have to interpolate them before flat-fielding."),
740 default=False,
741 )
742 maskListToInterpolate = pexConfig.ListField(
743 dtype=str,
744 doc="List of mask planes that should be interpolated.",
745 default=['SAT', 'BAD', 'UNMASKEDNAN'],
746 )
747 doSaveInterpPixels = pexConfig.Field(
748 dtype=bool,
749 doc="Save a copy of the pre-interpolated pixel values?",
750 default=False,
751 )
753 # Default photometric calibration options.
754 fluxMag0T1 = pexConfig.DictField(
755 keytype=str,
756 itemtype=float,
757 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
758 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
759 ))
760 )
761 defaultFluxMag0T1 = pexConfig.Field(
762 dtype=float,
763 doc="Default value for fluxMag0T1 (for an unrecognized filter).",
764 default=pow(10.0, 0.4*28.0)
765 )
767 # Vignette correction configuration.
768 doVignette = pexConfig.Field(
769 dtype=bool,
770 doc="Apply vignetting parameters?",
771 default=False,
772 )
773 vignette = pexConfig.ConfigurableField(
774 target=VignetteTask,
775 doc="Vignetting task.",
776 )
778 # Transmission curve configuration.
779 doAttachTransmissionCurve = pexConfig.Field(
780 dtype=bool,
781 default=False,
782 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
783 )
784 doUseOpticsTransmission = pexConfig.Field(
785 dtype=bool,
786 default=True,
787 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
788 )
789 doUseFilterTransmission = pexConfig.Field(
790 dtype=bool,
791 default=True,
792 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
793 )
794 doUseSensorTransmission = pexConfig.Field(
795 dtype=bool,
796 default=True,
797 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
798 )
799 doUseAtmosphereTransmission = pexConfig.Field(
800 dtype=bool,
801 default=True,
802 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
803 )
805 # Illumination correction.
806 doIlluminationCorrection = pexConfig.Field(
807 dtype=bool,
808 default=False,
809 doc="Perform illumination correction?"
810 )
811 illuminationCorrectionDataProductName = pexConfig.Field(
812 dtype=str,
813 doc="Name of the illumination correction data product.",
814 default="illumcor",
815 )
816 illumScale = pexConfig.Field(
817 dtype=float,
818 doc="Scale factor for the illumination correction.",
819 default=1.0,
820 )
821 illumFilters = pexConfig.ListField(
822 dtype=str,
823 default=[],
824 doc="Only perform illumination correction for these filters."
825 )
827 # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
828 doWrite = pexConfig.Field(
829 dtype=bool,
830 doc="Persist postISRCCD?",
831 default=True,
832 )
834 def validate(self):
835 super().validate()
836 if self.doFlat and self.doApplyGains:
837 raise ValueError("You may not specify both doFlat and doApplyGains")
838 if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate:
839 self.config.maskListToInterpolate.append("SAT")
840 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
841 self.config.maskListToInterpolate.append("UNMASKEDNAN")
844class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
845 """Apply common instrument signature correction algorithms to a raw frame.
847 The process for correcting imaging data is very similar from
848 camera to camera. This task provides a vanilla implementation of
849 doing these corrections, including the ability to turn certain
850 corrections off if they are not needed. The inputs to the primary
851 method, `run()`, are a raw exposure to be corrected and the
852 calibration data products. The raw input is a single chip sized
853 mosaic of all amps including overscans and other non-science
854 pixels. The method `runDataRef()` identifies and defines the
855 calibration data products, and is intended for use by a
856 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
857 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
858 subclassed for different camera, although the most camera specific
859 methods have been split into subtasks that can be redirected
860 appropriately.
862 The __init__ method sets up the subtasks for ISR processing, using
863 the defaults from `lsst.ip.isr`.
865 Parameters
866 ----------
867 args : `list`
868 Positional arguments passed to the Task constructor. None used at this time.
869 kwargs : `dict`, optional
870 Keyword arguments passed on to the Task constructor. None used at this time.
871 """
872 ConfigClass = IsrTaskConfig
873 _DefaultName = "isr"
875 def __init__(self, **kwargs):
876 super().__init__(**kwargs)
877 self.makeSubtask("assembleCcd")
878 self.makeSubtask("crosstalk")
879 self.makeSubtask("strayLight")
880 self.makeSubtask("fringe")
881 self.makeSubtask("masking")
882 self.makeSubtask("overscan")
883 self.makeSubtask("vignette")
885 def runQuantum(self, butlerQC, inputRefs, outputRefs):
886 inputs = butlerQC.get(inputRefs)
888 try:
889 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
890 except Exception as e:
891 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
892 (inputRefs, e))
894 inputs['isGen3'] = True
896 detector = inputs['ccdExposure'].getDetector()
898 if self.config.doCrosstalk is True:
899 # Crosstalk sources need to be defined by the pipeline
900 # yaml if they exist.
901 if 'crosstalk' in inputs and inputs['crosstalk'] is not None:
902 if not isinstance(inputs['crosstalk'], CrosstalkCalib):
903 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk'])
904 else:
905 coeffVector = (self.config.crosstalk.crosstalkValues
906 if self.config.crosstalk.useConfigCoefficients else None)
907 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
908 inputs['crosstalk'] = crosstalkCalib
909 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0:
910 if 'crosstalkSources' not in inputs:
911 self.log.warn("No crosstalkSources found for chip with interChip terms!")
913 if self.doLinearize(detector) is True:
914 if 'linearizer' in inputs and isinstance(inputs['linearizer'], dict):
915 linearizer = linearize.Linearizer(detector=detector, log=self.log)
916 linearizer.fromYaml(inputs['linearizer'])
917 else:
918 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), detector=detector,
919 log=self.log)
920 inputs['linearizer'] = linearizer
922 if self.config.doDefect is True:
923 if "defects" in inputs and inputs['defects'] is not None:
924 # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
925 # masking expects a list of defects defined by their bounding box
926 if not isinstance(inputs["defects"], Defects):
927 inputs["defects"] = Defects.fromTable(inputs["defects"])
929 # Load the correct style of brighter fatter kernel, and repack
930 # the information as a numpy array.
931 if self.config.doBrighterFatter:
932 brighterFatterKernel = inputs.pop('newBFKernel', None)
933 if brighterFatterKernel is None:
934 brighterFatterKernel = inputs.get('bfKernel', None)
936 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
937 detId = detector.getId()
938 inputs['bfGains'] = brighterFatterKernel.gain
939 # If the kernel is not an ndarray, it's the cp_pipe version
940 # so extract the kernel for this detector, or raise an error
941 if self.config.brighterFatterLevel == 'DETECTOR':
942 if brighterFatterKernel.detectorKernel:
943 inputs['bfKernel'] = brighterFatterKernel.detectorKernel[detId]
944 elif brighterFatterKernel.detectorKernelFromAmpKernels:
945 inputs['bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
946 else:
947 raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
948 else:
949 # TODO DM-15631 for implementing this
950 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
952 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
953 expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
954 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
955 expId=expId,
956 assembler=self.assembleCcd
957 if self.config.doAssembleIsrExposures else None)
958 else:
959 inputs['fringes'] = pipeBase.Struct(fringes=None)
961 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
962 if 'strayLightData' not in inputs:
963 inputs['strayLightData'] = None
965 outputs = self.run(**inputs)
966 butlerQC.put(outputs, outputRefs)
968 def readIsrData(self, dataRef, rawExposure):
969 """!Retrieve necessary frames for instrument signature removal.
971 Pre-fetching all required ISR data products limits the IO
972 required by the ISR. Any conflict between the calibration data
973 available and that needed for ISR is also detected prior to
974 doing processing, allowing it to fail quickly.
976 Parameters
977 ----------
978 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
979 Butler reference of the detector data to be processed
980 rawExposure : `afw.image.Exposure`
981 The raw exposure that will later be corrected with the
982 retrieved calibration data; should not be modified in this
983 method.
985 Returns
986 -------
987 result : `lsst.pipe.base.Struct`
988 Result struct with components (which may be `None`):
989 - ``bias``: bias calibration frame (`afw.image.Exposure`)
990 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
991 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
992 - ``dark``: dark calibration frame (`afw.image.Exposure`)
993 - ``flat``: flat calibration frame (`afw.image.Exposure`)
994 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
995 - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
996 - ``fringes``: `lsst.pipe.base.Struct` with components:
997 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
998 - ``seed``: random seed derived from the ccdExposureId for random
999 number generator (`uint32`).
1000 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1001 A ``TransmissionCurve`` that represents the throughput of the optics,
1002 to be evaluated in focal-plane coordinates.
1003 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1004 A ``TransmissionCurve`` that represents the throughput of the filter
1005 itself, to be evaluated in focal-plane coordinates.
1006 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1007 A ``TransmissionCurve`` that represents the throughput of the sensor
1008 itself, to be evaluated in post-assembly trimmed detector coordinates.
1009 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1010 A ``TransmissionCurve`` that represents the throughput of the
1011 atmosphere, assumed to be spatially constant.
1012 - ``strayLightData`` : `object`
1013 An opaque object containing calibration information for
1014 stray-light correction. If `None`, no correction will be
1015 performed.
1016 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1018 Raises
1019 ------
1020 NotImplementedError :
1021 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1022 """
1023 try:
1024 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1025 dateObs = dateObs.toPython().isoformat()
1026 except RuntimeError:
1027 self.log.warn("Unable to identify dateObs for rawExposure.")
1028 dateObs = None
1030 ccd = rawExposure.getDetector()
1031 filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
1032 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
1033 biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
1034 if self.config.doBias else None)
1035 # immediate=True required for functors and linearizers are functors; see ticket DM-6515
1036 linearizer = (dataRef.get("linearizer", immediate=True)
1037 if self.doLinearize(ccd) else None)
1038 if linearizer is not None and not isinstance(linearizer, numpy.ndarray):
1039 linearizer.log = self.log
1040 if isinstance(linearizer, numpy.ndarray):
1041 linearizer = linearize.Linearizer(table=linearizer, detector=ccd)
1043 crosstalkCalib = None
1044 if self.config.doCrosstalk:
1045 try:
1046 crosstalkCalib = dataRef.get("crosstalk", immediate=True)
1047 except NoResults:
1048 coeffVector = (self.config.crosstalk.crosstalkValues
1049 if self.config.crosstalk.useConfigCoefficients else None)
1050 crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1051 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1052 if self.config.doCrosstalk else None)
1054 darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
1055 if self.config.doDark else None)
1056 flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName,
1057 dateObs=dateObs)
1058 if self.config.doFlat else None)
1060 brighterFatterKernel = None
1061 brighterFatterGains = None
1062 if self.config.doBrighterFatter is True:
1063 try:
1064 # Use the new-style cp_pipe version of the kernel if it exists
1065 # If using a new-style kernel, always use the self-consistent
1066 # gains, i.e. the ones inside the kernel object itself
1067 brighterFatterKernel = dataRef.get("brighterFatterKernel")
1068 brighterFatterGains = brighterFatterKernel.gain
1069 self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded")
1070 except NoResults:
1071 try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
1072 brighterFatterKernel = dataRef.get("bfKernel")
1073 self.log.info("Old style bright-fatter kernel (np.array) loaded")
1074 except NoResults:
1075 brighterFatterKernel = None
1076 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1077 # If the kernel is not an ndarray, it's the cp_pipe version
1078 # so extract the kernel for this detector, or raise an error
1079 if self.config.brighterFatterLevel == 'DETECTOR':
1080 if brighterFatterKernel.detectorKernel:
1081 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
1082 elif brighterFatterKernel.detectorKernelFromAmpKernels:
1083 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
1084 else:
1085 raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1086 else:
1087 # TODO DM-15631 for implementing this
1088 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1090 defectList = (dataRef.get("defects")
1091 if self.config.doDefect else None)
1092 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
1093 if self.config.doAssembleIsrExposures else None)
1094 if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1095 else pipeBase.Struct(fringes=None))
1097 if self.config.doAttachTransmissionCurve:
1098 opticsTransmission = (dataRef.get("transmission_optics")
1099 if self.config.doUseOpticsTransmission else None)
1100 filterTransmission = (dataRef.get("transmission_filter")
1101 if self.config.doUseFilterTransmission else None)
1102 sensorTransmission = (dataRef.get("transmission_sensor")
1103 if self.config.doUseSensorTransmission else None)
1104 atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1105 if self.config.doUseAtmosphereTransmission else None)
1106 else:
1107 opticsTransmission = None
1108 filterTransmission = None
1109 sensorTransmission = None
1110 atmosphereTransmission = None
1112 if self.config.doStrayLight:
1113 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1114 else:
1115 strayLightData = None
1117 illumMaskedImage = (self.getIsrExposure(dataRef,
1118 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1119 if (self.config.doIlluminationCorrection and
1120 filterName in self.config.illumFilters)
1121 else None)
1123 # Struct should include only kwargs to run()
1124 return pipeBase.Struct(bias=biasExposure,
1125 linearizer=linearizer,
1126 crosstalk=crosstalkCalib,
1127 crosstalkSources=crosstalkSources,
1128 dark=darkExposure,
1129 flat=flatExposure,
1130 bfKernel=brighterFatterKernel,
1131 bfGains=brighterFatterGains,
1132 defects=defectList,
1133 fringes=fringeStruct,
1134 opticsTransmission=opticsTransmission,
1135 filterTransmission=filterTransmission,
1136 sensorTransmission=sensorTransmission,
1137 atmosphereTransmission=atmosphereTransmission,
1138 strayLightData=strayLightData,
1139 illumMaskedImage=illumMaskedImage
1140 )
1142 @pipeBase.timeMethod
1143 def run(self, ccdExposure, camera=None, bias=None, linearizer=None,
1144 crosstalk=None, crosstalkSources=None,
1145 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1146 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1147 sensorTransmission=None, atmosphereTransmission=None,
1148 detectorNum=None, strayLightData=None, illumMaskedImage=None,
1149 isGen3=False,
1150 ):
1151 """!Perform instrument signature removal on an exposure.
1153 Steps included in the ISR processing, in order performed, are:
1154 - saturation and suspect pixel masking
1155 - overscan subtraction
1156 - CCD assembly of individual amplifiers
1157 - bias subtraction
1158 - variance image construction
1159 - linearization of non-linear response
1160 - crosstalk masking
1161 - brighter-fatter correction
1162 - dark subtraction
1163 - fringe correction
1164 - stray light subtraction
1165 - flat correction
1166 - masking of known defects and camera specific features
1167 - vignette calculation
1168 - appending transmission curve and distortion model
1170 Parameters
1171 ----------
1172 ccdExposure : `lsst.afw.image.Exposure`
1173 The raw exposure that is to be run through ISR. The
1174 exposure is modified by this method.
1175 camera : `lsst.afw.cameraGeom.Camera`, optional
1176 The camera geometry for this exposure. Required if ``isGen3`` is
1177 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1178 ``flat`` does not have an associated detector.
1179 bias : `lsst.afw.image.Exposure`, optional
1180 Bias calibration frame.
1181 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1182 Functor for linearization.
1183 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1184 Calibration for crosstalk.
1185 crosstalkSources : `list`, optional
1186 List of possible crosstalk sources.
1187 dark : `lsst.afw.image.Exposure`, optional
1188 Dark calibration frame.
1189 flat : `lsst.afw.image.Exposure`, optional
1190 Flat calibration frame.
1191 bfKernel : `numpy.ndarray`, optional
1192 Brighter-fatter kernel.
1193 bfGains : `dict` of `float`, optional
1194 Gains used to override the detector's nominal gains for the
1195 brighter-fatter correction. A dict keyed by amplifier name for
1196 the detector in question.
1197 defects : `lsst.meas.algorithms.Defects`, optional
1198 List of defects.
1199 fringes : `lsst.pipe.base.Struct`, optional
1200 Struct containing the fringe correction data, with
1201 elements:
1202 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1203 - ``seed``: random seed derived from the ccdExposureId for random
1204 number generator (`uint32`)
1205 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1206 A ``TransmissionCurve`` that represents the throughput of the optics,
1207 to be evaluated in focal-plane coordinates.
1208 filterTransmission : `lsst.afw.image.TransmissionCurve`
1209 A ``TransmissionCurve`` that represents the throughput of the filter
1210 itself, to be evaluated in focal-plane coordinates.
1211 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1212 A ``TransmissionCurve`` that represents the throughput of the sensor
1213 itself, to be evaluated in post-assembly trimmed detector coordinates.
1214 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1215 A ``TransmissionCurve`` that represents the throughput of the
1216 atmosphere, assumed to be spatially constant.
1217 detectorNum : `int`, optional
1218 The integer number for the detector to process.
1219 isGen3 : bool, optional
1220 Flag this call to run() as using the Gen3 butler environment.
1221 strayLightData : `object`, optional
1222 Opaque object containing calibration information for stray-light
1223 correction. If `None`, no correction will be performed.
1224 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1225 Illumination correction image.
1227 Returns
1228 -------
1229 result : `lsst.pipe.base.Struct`
1230 Result struct with component:
1231 - ``exposure`` : `afw.image.Exposure`
1232 The fully ISR corrected exposure.
1233 - ``outputExposure`` : `afw.image.Exposure`
1234 An alias for `exposure`
1235 - ``ossThumb`` : `numpy.ndarray`
1236 Thumbnail image of the exposure after overscan subtraction.
1237 - ``flattenedThumb`` : `numpy.ndarray`
1238 Thumbnail image of the exposure after flat-field correction.
1240 Raises
1241 ------
1242 RuntimeError
1243 Raised if a configuration option is set to True, but the
1244 required calibration data has not been specified.
1246 Notes
1247 -----
1248 The current processed exposure can be viewed by setting the
1249 appropriate lsstDebug entries in the `debug.display`
1250 dictionary. The names of these entries correspond to some of
1251 the IsrTaskConfig Boolean options, with the value denoting the
1252 frame to use. The exposure is shown inside the matching
1253 option check and after the processing of that step has
1254 finished. The steps with debug points are:
1256 doAssembleCcd
1257 doBias
1258 doCrosstalk
1259 doBrighterFatter
1260 doDark
1261 doFringe
1262 doStrayLight
1263 doFlat
1265 In addition, setting the "postISRCCD" entry displays the
1266 exposure after all ISR processing has finished.
1268 """
1270 if isGen3 is True:
1271 # Gen3 currently cannot automatically do configuration overrides.
1272 # DM-15257 looks to discuss this issue.
1273 # Configure input exposures;
1274 if detectorNum is None:
1275 raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1277 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1278 bias = self.ensureExposure(bias, camera, detectorNum)
1279 dark = self.ensureExposure(dark, camera, detectorNum)
1280 flat = self.ensureExposure(flat, camera, detectorNum)
1281 else:
1282 if isinstance(ccdExposure, ButlerDataRef):
1283 return self.runDataRef(ccdExposure)
1285 ccd = ccdExposure.getDetector()
1286 filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1288 if not ccd:
1289 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1290 ccd = [FakeAmp(ccdExposure, self.config)]
1292 # Validate Input
1293 if self.config.doBias and bias is None:
1294 raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1295 if self.doLinearize(ccd) and linearizer is None:
1296 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1297 if self.config.doBrighterFatter and bfKernel is None:
1298 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1299 if self.config.doDark and dark is None:
1300 raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1301 if self.config.doFlat and flat is None:
1302 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1303 if self.config.doDefect and defects is None:
1304 raise RuntimeError("Must supply defects if config.doDefect=True.")
1305 if (self.config.doFringe and filterName in self.fringe.config.filters and
1306 fringes.fringes is None):
1307 # The `fringes` object needs to be a pipeBase.Struct, as
1308 # we use it as a `dict` for the parameters of
1309 # `FringeTask.run()`. The `fringes.fringes` `list` may
1310 # not be `None` if `doFringe=True`. Otherwise, raise.
1311 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1312 if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters and
1313 illumMaskedImage is None):
1314 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1316 # Begin ISR processing.
1317 if self.config.doConvertIntToFloat:
1318 self.log.info("Converting exposure to floating point values.")
1319 ccdExposure = self.convertIntToFloat(ccdExposure)
1321 # Amplifier level processing.
1322 overscans = []
1323 for amp in ccd:
1324 # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1325 if ccdExposure.getBBox().contains(amp.getBBox()):
1326 # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1327 badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1329 if self.config.doOverscan and not badAmp:
1330 # Overscan correction on amp-by-amp basis.
1331 overscanResults = self.overscanCorrection(ccdExposure, amp)
1332 self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1333 if overscanResults is not None and \
1334 self.config.qa is not None and self.config.qa.saveStats is True:
1335 if isinstance(overscanResults.overscanFit, float):
1336 qaMedian = overscanResults.overscanFit
1337 qaStdev = float("NaN")
1338 else:
1339 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1340 afwMath.MEDIAN | afwMath.STDEVCLIP)
1341 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1342 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1344 self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1345 self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1346 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1347 amp.getName(), qaMedian, qaStdev)
1348 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1349 else:
1350 if badAmp:
1351 self.log.warn("Amplifier %s is bad.", amp.getName())
1352 overscanResults = None
1354 overscans.append(overscanResults if overscanResults is not None else None)
1355 else:
1356 self.log.info("Skipped OSCAN for %s.", amp.getName())
1358 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1359 self.log.info("Applying crosstalk correction.")
1360 self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1361 crosstalkSources=crosstalkSources)
1362 self.debugView(ccdExposure, "doCrosstalk")
1364 if self.config.doAssembleCcd:
1365 self.log.info("Assembling CCD from amplifiers.")
1366 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1368 if self.config.expectWcs and not ccdExposure.getWcs():
1369 self.log.warn("No WCS found in input exposure.")
1370 self.debugView(ccdExposure, "doAssembleCcd")
1372 ossThumb = None
1373 if self.config.qa.doThumbnailOss:
1374 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1376 if self.config.doBias:
1377 self.log.info("Applying bias correction.")
1378 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1379 trimToFit=self.config.doTrimToMatchCalib)
1380 self.debugView(ccdExposure, "doBias")
1382 if self.config.doVariance:
1383 for amp, overscanResults in zip(ccd, overscans):
1384 if ccdExposure.getBBox().contains(amp.getBBox()):
1385 self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1386 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1387 if overscanResults is not None:
1388 self.updateVariance(ampExposure, amp,
1389 overscanImage=overscanResults.overscanImage)
1390 else:
1391 self.updateVariance(ampExposure, amp,
1392 overscanImage=None)
1393 if self.config.qa is not None and self.config.qa.saveStats is True:
1394 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1395 afwMath.MEDIAN | afwMath.STDEVCLIP)
1396 self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1397 qaStats.getValue(afwMath.MEDIAN))
1398 self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1399 qaStats.getValue(afwMath.STDEVCLIP))
1400 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1401 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1402 qaStats.getValue(afwMath.STDEVCLIP))
1404 if self.doLinearize(ccd):
1405 self.log.info("Applying linearizer.")
1406 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1407 detector=ccd, log=self.log)
1409 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1410 self.log.info("Applying crosstalk correction.")
1411 self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1412 crosstalkSources=crosstalkSources, isTrimmed=True)
1413 self.debugView(ccdExposure, "doCrosstalk")
1415 # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1416 # anything else the camera needs. Saturated and suspect pixels have already been masked.
1417 if self.config.doDefect:
1418 self.log.info("Masking defects.")
1419 self.maskDefect(ccdExposure, defects)
1421 if self.config.numEdgeSuspect > 0:
1422 self.log.info("Masking edges as SUSPECT.")
1423 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1424 maskPlane="SUSPECT")
1426 if self.config.doNanMasking:
1427 self.log.info("Masking NAN value pixels.")
1428 self.maskNan(ccdExposure)
1430 if self.config.doWidenSaturationTrails:
1431 self.log.info("Widening saturation trails.")
1432 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1434 if self.config.doCameraSpecificMasking:
1435 self.log.info("Masking regions for camera specific reasons.")
1436 self.masking.run(ccdExposure)
1438 if self.config.doBrighterFatter:
1439 # We need to apply flats and darks before we can interpolate, and we
1440 # need to interpolate before we do B-F, but we do B-F without the
1441 # flats and darks applied so we can work in units of electrons or holes.
1442 # This context manager applies and then removes the darks and flats.
1443 #
1444 # We also do not want to interpolate values here, so operate on temporary
1445 # images so we can apply only the BF-correction and roll back the
1446 # interpolation.
1447 interpExp = ccdExposure.clone()
1448 with self.flatContext(interpExp, flat, dark):
1449 isrFunctions.interpolateFromMask(
1450 maskedImage=interpExp.getMaskedImage(),
1451 fwhm=self.config.fwhm,
1452 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1453 maskNameList=self.config.maskListToInterpolate
1454 )
1455 bfExp = interpExp.clone()
1457 self.log.info("Applying brighter fatter correction using kernel type %s / gains %s.",
1458 type(bfKernel), type(bfGains))
1459 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1460 self.config.brighterFatterMaxIter,
1461 self.config.brighterFatterThreshold,
1462 self.config.brighterFatterApplyGain,
1463 bfGains)
1464 if bfResults[1] == self.config.brighterFatterMaxIter:
1465 self.log.warn("Brighter fatter correction did not converge, final difference %f.",
1466 bfResults[0])
1467 else:
1468 self.log.info("Finished brighter fatter correction in %d iterations.",
1469 bfResults[1])
1470 image = ccdExposure.getMaskedImage().getImage()
1471 bfCorr = bfExp.getMaskedImage().getImage()
1472 bfCorr -= interpExp.getMaskedImage().getImage()
1473 image += bfCorr
1475 # Applying the brighter-fatter correction applies a
1476 # convolution to the science image. At the edges this
1477 # convolution may not have sufficient valid pixels to
1478 # produce a valid correction. Mark pixels within the size
1479 # of the brighter-fatter kernel as EDGE to warn of this
1480 # fact.
1481 self.log.info("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1482 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1483 maskPlane="EDGE")
1485 if self.config.brighterFatterMaskGrowSize > 0:
1486 self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1487 for maskPlane in self.config.maskListToInterpolate:
1488 isrFunctions.growMasks(ccdExposure.getMask(),
1489 radius=self.config.brighterFatterMaskGrowSize,
1490 maskNameList=maskPlane,
1491 maskValue=maskPlane)
1493 self.debugView(ccdExposure, "doBrighterFatter")
1495 if self.config.doDark:
1496 self.log.info("Applying dark correction.")
1497 self.darkCorrection(ccdExposure, dark)
1498 self.debugView(ccdExposure, "doDark")
1500 if self.config.doFringe and not self.config.fringeAfterFlat:
1501 self.log.info("Applying fringe correction before flat.")
1502 self.fringe.run(ccdExposure, **fringes.getDict())
1503 self.debugView(ccdExposure, "doFringe")
1505 if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1506 self.log.info("Checking strayLight correction.")
1507 self.strayLight.run(ccdExposure, strayLightData)
1508 self.debugView(ccdExposure, "doStrayLight")
1510 if self.config.doFlat:
1511 self.log.info("Applying flat correction.")
1512 self.flatCorrection(ccdExposure, flat)
1513 self.debugView(ccdExposure, "doFlat")
1515 if self.config.doApplyGains:
1516 self.log.info("Applying gain correction instead of flat.")
1517 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1519 if self.config.doFringe and self.config.fringeAfterFlat:
1520 self.log.info("Applying fringe correction after flat.")
1521 self.fringe.run(ccdExposure, **fringes.getDict())
1523 if self.config.doVignette:
1524 self.log.info("Constructing Vignette polygon.")
1525 self.vignettePolygon = self.vignette.run(ccdExposure)
1527 if self.config.vignette.doWriteVignettePolygon:
1528 self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1530 if self.config.doAttachTransmissionCurve:
1531 self.log.info("Adding transmission curves.")
1532 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1533 filterTransmission=filterTransmission,
1534 sensorTransmission=sensorTransmission,
1535 atmosphereTransmission=atmosphereTransmission)
1537 flattenedThumb = None
1538 if self.config.qa.doThumbnailFlattened:
1539 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1541 if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1542 self.log.info("Performing illumination correction.")
1543 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1544 illumMaskedImage, illumScale=self.config.illumScale,
1545 trimToFit=self.config.doTrimToMatchCalib)
1547 preInterpExp = None
1548 if self.config.doSaveInterpPixels:
1549 preInterpExp = ccdExposure.clone()
1551 # Reset and interpolate bad pixels.
1552 #
1553 # Large contiguous bad regions (which should have the BAD mask
1554 # bit set) should have their values set to the image median.
1555 # This group should include defects and bad amplifiers. As the
1556 # area covered by these defects are large, there's little
1557 # reason to expect that interpolation would provide a more
1558 # useful value.
1559 #
1560 # Smaller defects can be safely interpolated after the larger
1561 # regions have had their pixel values reset. This ensures
1562 # that the remaining defects adjacent to bad amplifiers (as an
1563 # example) do not attempt to interpolate extreme values.
1564 if self.config.doSetBadRegions:
1565 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1566 if badPixelCount > 0:
1567 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1569 if self.config.doInterpolate:
1570 self.log.info("Interpolating masked pixels.")
1571 isrFunctions.interpolateFromMask(
1572 maskedImage=ccdExposure.getMaskedImage(),
1573 fwhm=self.config.fwhm,
1574 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1575 maskNameList=list(self.config.maskListToInterpolate)
1576 )
1578 self.roughZeroPoint(ccdExposure)
1580 if self.config.doMeasureBackground:
1581 self.log.info("Measuring background level.")
1582 self.measureBackground(ccdExposure, self.config.qa)
1584 if self.config.qa is not None and self.config.qa.saveStats is True:
1585 for amp in ccd:
1586 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1587 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1588 afwMath.MEDIAN | afwMath.STDEVCLIP)
1589 self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1590 qaStats.getValue(afwMath.MEDIAN))
1591 self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1592 qaStats.getValue(afwMath.STDEVCLIP))
1593 self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1594 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1595 qaStats.getValue(afwMath.STDEVCLIP))
1597 self.debugView(ccdExposure, "postISRCCD")
1599 return pipeBase.Struct(
1600 exposure=ccdExposure,
1601 ossThumb=ossThumb,
1602 flattenedThumb=flattenedThumb,
1604 preInterpolatedExposure=preInterpExp,
1605 outputExposure=ccdExposure,
1606 outputOssThumbnail=ossThumb,
1607 outputFlattenedThumbnail=flattenedThumb,
1608 )
1610 @pipeBase.timeMethod
1611 def runDataRef(self, sensorRef):
1612 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1614 This method contains the `CmdLineTask` interface to the ISR
1615 processing. All IO is handled here, freeing the `run()` method
1616 to manage only pixel-level calculations. The steps performed
1617 are:
1618 - Read in necessary detrending/isr/calibration data.
1619 - Process raw exposure in `run()`.
1620 - Persist the ISR-corrected exposure as "postISRCCD" if
1621 config.doWrite=True.
1623 Parameters
1624 ----------
1625 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1626 DataRef of the detector data to be processed
1628 Returns
1629 -------
1630 result : `lsst.pipe.base.Struct`
1631 Result struct with component:
1632 - ``exposure`` : `afw.image.Exposure`
1633 The fully ISR corrected exposure.
1635 Raises
1636 ------
1637 RuntimeError
1638 Raised if a configuration option is set to True, but the
1639 required calibration data does not exist.
1641 """
1642 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1644 ccdExposure = sensorRef.get(self.config.datasetType)
1646 camera = sensorRef.get("camera")
1647 isrData = self.readIsrData(sensorRef, ccdExposure)
1649 result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1651 if self.config.doWrite:
1652 sensorRef.put(result.exposure, "postISRCCD")
1653 if result.preInterpolatedExposure is not None:
1654 sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1655 if result.ossThumb is not None:
1656 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1657 if result.flattenedThumb is not None:
1658 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1660 return result
1662 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1663 """!Retrieve a calibration dataset for removing instrument signature.
1665 Parameters
1666 ----------
1668 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1669 DataRef of the detector data to find calibration datasets
1670 for.
1671 datasetType : `str`
1672 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1673 dateObs : `str`, optional
1674 Date of the observation. Used to correct butler failures
1675 when using fallback filters.
1676 immediate : `Bool`
1677 If True, disable butler proxies to enable error handling
1678 within this routine.
1680 Returns
1681 -------
1682 exposure : `lsst.afw.image.Exposure`
1683 Requested calibration frame.
1685 Raises
1686 ------
1687 RuntimeError
1688 Raised if no matching calibration frame can be found.
1689 """
1690 try:
1691 exp = dataRef.get(datasetType, immediate=immediate)
1692 except Exception as exc1:
1693 if not self.config.fallbackFilterName:
1694 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1695 try:
1696 if self.config.useFallbackDate and dateObs:
1697 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1698 dateObs=dateObs, immediate=immediate)
1699 else:
1700 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1701 except Exception as exc2:
1702 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1703 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1704 self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1706 if self.config.doAssembleIsrExposures:
1707 exp = self.assembleCcd.assembleCcd(exp)
1708 return exp
1710 def ensureExposure(self, inputExp, camera, detectorNum):
1711 """Ensure that the data returned by Butler is a fully constructed exposure.
1713 ISR requires exposure-level image data for historical reasons, so if we did
1714 not recieve that from Butler, construct it from what we have, modifying the
1715 input in place.
1717 Parameters
1718 ----------
1719 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1720 `lsst.afw.image.ImageF`
1721 The input data structure obtained from Butler.
1722 camera : `lsst.afw.cameraGeom.camera`
1723 The camera associated with the image. Used to find the appropriate
1724 detector.
1725 detectorNum : `int`
1726 The detector this exposure should match.
1728 Returns
1729 -------
1730 inputExp : `lsst.afw.image.Exposure`
1731 The re-constructed exposure, with appropriate detector parameters.
1733 Raises
1734 ------
1735 TypeError
1736 Raised if the input data cannot be used to construct an exposure.
1737 """
1738 if isinstance(inputExp, afwImage.DecoratedImageU):
1739 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1740 elif isinstance(inputExp, afwImage.ImageF):
1741 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1742 elif isinstance(inputExp, afwImage.MaskedImageF):
1743 inputExp = afwImage.makeExposure(inputExp)
1744 elif isinstance(inputExp, afwImage.Exposure):
1745 pass
1746 elif inputExp is None:
1747 # Assume this will be caught by the setup if it is a problem.
1748 return inputExp
1749 else:
1750 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1751 (type(inputExp), ))
1753 if inputExp.getDetector() is None:
1754 inputExp.setDetector(camera[detectorNum])
1756 return inputExp
1758 def convertIntToFloat(self, exposure):
1759 """Convert exposure image from uint16 to float.
1761 If the exposure does not need to be converted, the input is
1762 immediately returned. For exposures that are converted to use
1763 floating point pixels, the variance is set to unity and the
1764 mask to zero.
1766 Parameters
1767 ----------
1768 exposure : `lsst.afw.image.Exposure`
1769 The raw exposure to be converted.
1771 Returns
1772 -------
1773 newexposure : `lsst.afw.image.Exposure`
1774 The input ``exposure``, converted to floating point pixels.
1776 Raises
1777 ------
1778 RuntimeError
1779 Raised if the exposure type cannot be converted to float.
1781 """
1782 if isinstance(exposure, afwImage.ExposureF):
1783 # Nothing to be done
1784 self.log.debug("Exposure already of type float.")
1785 return exposure
1786 if not hasattr(exposure, "convertF"):
1787 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1789 newexposure = exposure.convertF()
1790 newexposure.variance[:] = 1
1791 newexposure.mask[:] = 0x0
1793 return newexposure
1795 def maskAmplifier(self, ccdExposure, amp, defects):
1796 """Identify bad amplifiers, saturated and suspect pixels.
1798 Parameters
1799 ----------
1800 ccdExposure : `lsst.afw.image.Exposure`
1801 Input exposure to be masked.
1802 amp : `lsst.afw.table.AmpInfoCatalog`
1803 Catalog of parameters defining the amplifier on this
1804 exposure to mask.
1805 defects : `lsst.meas.algorithms.Defects`
1806 List of defects. Used to determine if the entire
1807 amplifier is bad.
1809 Returns
1810 -------
1811 badAmp : `Bool`
1812 If this is true, the entire amplifier area is covered by
1813 defects and unusable.
1815 """
1816 maskedImage = ccdExposure.getMaskedImage()
1818 badAmp = False
1820 # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1821 # comparison with current defects definition.
1822 if defects is not None:
1823 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1825 # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1826 # association with pixels in current ccdExposure).
1827 if badAmp:
1828 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1829 afwImage.PARENT)
1830 maskView = dataView.getMask()
1831 maskView |= maskView.getPlaneBitMask("BAD")
1832 del maskView
1833 return badAmp
1835 # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1836 # Saturation and suspect pixels can be masked now, though.
1837 limits = dict()
1838 if self.config.doSaturation and not badAmp:
1839 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1840 if self.config.doSuspect and not badAmp:
1841 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1842 if math.isfinite(self.config.saturation):
1843 limits.update({self.config.saturatedMaskName: self.config.saturation})
1845 for maskName, maskThreshold in limits.items():
1846 if not math.isnan(maskThreshold):
1847 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1848 isrFunctions.makeThresholdMask(
1849 maskedImage=dataView,
1850 threshold=maskThreshold,
1851 growFootprints=0,
1852 maskName=maskName
1853 )
1855 # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1856 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1857 afwImage.PARENT)
1858 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1859 self.config.suspectMaskName])
1860 if numpy.all(maskView.getArray() & maskVal > 0):
1861 badAmp = True
1862 maskView |= maskView.getPlaneBitMask("BAD")
1864 return badAmp
1866 def overscanCorrection(self, ccdExposure, amp):
1867 """Apply overscan correction in place.
1869 This method does initial pixel rejection of the overscan
1870 region. The overscan can also be optionally segmented to
1871 allow for discontinuous overscan responses to be fit
1872 separately. The actual overscan subtraction is performed by
1873 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1874 which is called here after the amplifier is preprocessed.
1876 Parameters
1877 ----------
1878 ccdExposure : `lsst.afw.image.Exposure`
1879 Exposure to have overscan correction performed.
1880 amp : `lsst.afw.table.AmpInfoCatalog`
1881 The amplifier to consider while correcting the overscan.
1883 Returns
1884 -------
1885 overscanResults : `lsst.pipe.base.Struct`
1886 Result struct with components:
1887 - ``imageFit`` : scalar or `lsst.afw.image.Image`
1888 Value or fit subtracted from the amplifier image data.
1889 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1890 Value or fit subtracted from the overscan image data.
1891 - ``overscanImage`` : `lsst.afw.image.Image`
1892 Image of the overscan region with the overscan
1893 correction applied. This quantity is used to estimate
1894 the amplifier read noise empirically.
1896 Raises
1897 ------
1898 RuntimeError
1899 Raised if the ``amp`` does not contain raw pixel information.
1901 See Also
1902 --------
1903 lsst.ip.isr.isrFunctions.overscanCorrection
1904 """
1905 if amp.getRawHorizontalOverscanBBox().isEmpty():
1906 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1907 return None
1909 statControl = afwMath.StatisticsControl()
1910 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1912 # Determine the bounding boxes
1913 dataBBox = amp.getRawDataBBox()
1914 oscanBBox = amp.getRawHorizontalOverscanBBox()
1915 dx0 = 0
1916 dx1 = 0
1918 prescanBBox = amp.getRawPrescanBBox()
1919 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1920 dx0 += self.config.overscanNumLeadingColumnsToSkip
1921 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1922 else:
1923 dx0 += self.config.overscanNumTrailingColumnsToSkip
1924 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1926 # Determine if we need to work on subregions of the amplifier and overscan.
1927 imageBBoxes = []
1928 overscanBBoxes = []
1930 if ((self.config.overscanBiasJump and
1931 self.config.overscanBiasJumpLocation) and
1932 (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword) and
1933 ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1934 self.config.overscanBiasJumpDevices)):
1935 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
1936 yLower = self.config.overscanBiasJumpLocation
1937 yUpper = dataBBox.getHeight() - yLower
1938 else:
1939 yUpper = self.config.overscanBiasJumpLocation
1940 yLower = dataBBox.getHeight() - yUpper
1942 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1943 lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1944 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() +
1945 lsst.geom.Extent2I(dx0, 0),
1946 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1947 yLower)))
1949 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1950 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1951 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1952 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1953 yUpper)))
1954 else:
1955 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1956 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1957 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1958 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1959 oscanBBox.getHeight())))
1961 # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1962 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1963 ampImage = ccdExposure.maskedImage[imageBBox]
1964 overscanImage = ccdExposure.maskedImage[overscanBBox]
1966 overscanArray = overscanImage.image.array
1967 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1968 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1969 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1971 statControl = afwMath.StatisticsControl()
1972 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1974 overscanResults = self.overscan.run(ampImage.getImage(), overscanImage)
1976 # Measure average overscan levels and record them in the metadata.
1977 levelStat = afwMath.MEDIAN
1978 sigmaStat = afwMath.STDEVCLIP
1980 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1981 self.config.qa.flatness.nIter)
1982 metadata = ccdExposure.getMetadata()
1983 ampNum = amp.getName()
1984 # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1985 if isinstance(overscanResults.overscanFit, float):
1986 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1987 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1988 else:
1989 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1990 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1991 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1993 return overscanResults
1995 def updateVariance(self, ampExposure, amp, overscanImage=None):
1996 """Set the variance plane using the amplifier gain and read noise
1998 The read noise is calculated from the ``overscanImage`` if the
1999 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2000 the value from the amplifier data is used.
2002 Parameters
2003 ----------
2004 ampExposure : `lsst.afw.image.Exposure`
2005 Exposure to process.
2006 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2007 Amplifier detector data.
2008 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2009 Image of overscan, required only for empirical read noise.
2011 See also
2012 --------
2013 lsst.ip.isr.isrFunctions.updateVariance
2014 """
2015 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2016 gain = amp.getGain()
2018 if math.isnan(gain):
2019 gain = 1.0
2020 self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2021 elif gain <= 0:
2022 patchedGain = 1.0
2023 self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
2024 amp.getName(), gain, patchedGain)
2025 gain = patchedGain
2027 if self.config.doEmpiricalReadNoise and overscanImage is None:
2028 self.log.info("Overscan is none for EmpiricalReadNoise.")
2030 if self.config.doEmpiricalReadNoise and overscanImage is not None:
2031 stats = afwMath.StatisticsControl()
2032 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2033 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2034 self.log.info("Calculated empirical read noise for amp %s: %f.",
2035 amp.getName(), readNoise)
2036 else:
2037 readNoise = amp.getReadNoise()
2039 isrFunctions.updateVariance(
2040 maskedImage=ampExposure.getMaskedImage(),
2041 gain=gain,
2042 readNoise=readNoise,
2043 )
2045 def darkCorrection(self, exposure, darkExposure, invert=False):
2046 """!Apply dark correction in place.
2048 Parameters
2049 ----------
2050 exposure : `lsst.afw.image.Exposure`
2051 Exposure to process.
2052 darkExposure : `lsst.afw.image.Exposure`
2053 Dark exposure of the same size as ``exposure``.
2054 invert : `Bool`, optional
2055 If True, re-add the dark to an already corrected image.
2057 Raises
2058 ------
2059 RuntimeError
2060 Raised if either ``exposure`` or ``darkExposure`` do not
2061 have their dark time defined.
2063 See Also
2064 --------
2065 lsst.ip.isr.isrFunctions.darkCorrection
2066 """
2067 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2068 if math.isnan(expScale):
2069 raise RuntimeError("Exposure darktime is NAN.")
2070 if darkExposure.getInfo().getVisitInfo() is not None \
2071 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2072 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2073 else:
2074 # DM-17444: darkExposure.getInfo.getVisitInfo() is None
2075 # so getDarkTime() does not exist.
2076 self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2077 darkScale = 1.0
2079 isrFunctions.darkCorrection(
2080 maskedImage=exposure.getMaskedImage(),
2081 darkMaskedImage=darkExposure.getMaskedImage(),
2082 expScale=expScale,
2083 darkScale=darkScale,
2084 invert=invert,
2085 trimToFit=self.config.doTrimToMatchCalib
2086 )
2088 def doLinearize(self, detector):
2089 """!Check if linearization is needed for the detector cameraGeom.
2091 Checks config.doLinearize and the linearity type of the first
2092 amplifier.
2094 Parameters
2095 ----------
2096 detector : `lsst.afw.cameraGeom.Detector`
2097 Detector to get linearity type from.
2099 Returns
2100 -------
2101 doLinearize : `Bool`
2102 If True, linearization should be performed.
2103 """
2104 return self.config.doLinearize and \
2105 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2107 def flatCorrection(self, exposure, flatExposure, invert=False):
2108 """!Apply flat correction in place.
2110 Parameters
2111 ----------
2112 exposure : `lsst.afw.image.Exposure`
2113 Exposure to process.
2114 flatExposure : `lsst.afw.image.Exposure`
2115 Flat exposure of the same size as ``exposure``.
2116 invert : `Bool`, optional
2117 If True, unflatten an already flattened image.
2119 See Also
2120 --------
2121 lsst.ip.isr.isrFunctions.flatCorrection
2122 """
2123 isrFunctions.flatCorrection(
2124 maskedImage=exposure.getMaskedImage(),
2125 flatMaskedImage=flatExposure.getMaskedImage(),
2126 scalingType=self.config.flatScalingType,
2127 userScale=self.config.flatUserScale,
2128 invert=invert,
2129 trimToFit=self.config.doTrimToMatchCalib
2130 )
2132 def saturationDetection(self, exposure, amp):
2133 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2135 Parameters
2136 ----------
2137 exposure : `lsst.afw.image.Exposure`
2138 Exposure to process. Only the amplifier DataSec is processed.
2139 amp : `lsst.afw.table.AmpInfoCatalog`
2140 Amplifier detector data.
2142 See Also
2143 --------
2144 lsst.ip.isr.isrFunctions.makeThresholdMask
2145 """
2146 if not math.isnan(amp.getSaturation()):
2147 maskedImage = exposure.getMaskedImage()
2148 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2149 isrFunctions.makeThresholdMask(
2150 maskedImage=dataView,
2151 threshold=amp.getSaturation(),
2152 growFootprints=0,
2153 maskName=self.config.saturatedMaskName,
2154 )
2156 def saturationInterpolation(self, exposure):
2157 """!Interpolate over saturated pixels, in place.
2159 This method should be called after `saturationDetection`, to
2160 ensure that the saturated pixels have been identified in the
2161 SAT mask. It should also be called after `assembleCcd`, since
2162 saturated regions may cross amplifier boundaries.
2164 Parameters
2165 ----------
2166 exposure : `lsst.afw.image.Exposure`
2167 Exposure to process.
2169 See Also
2170 --------
2171 lsst.ip.isr.isrTask.saturationDetection
2172 lsst.ip.isr.isrFunctions.interpolateFromMask
2173 """
2174 isrFunctions.interpolateFromMask(
2175 maskedImage=exposure.getMaskedImage(),
2176 fwhm=self.config.fwhm,
2177 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2178 maskNameList=list(self.config.saturatedMaskName),
2179 )
2181 def suspectDetection(self, exposure, amp):
2182 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2184 Parameters
2185 ----------
2186 exposure : `lsst.afw.image.Exposure`
2187 Exposure to process. Only the amplifier DataSec is processed.
2188 amp : `lsst.afw.table.AmpInfoCatalog`
2189 Amplifier detector data.
2191 See Also
2192 --------
2193 lsst.ip.isr.isrFunctions.makeThresholdMask
2195 Notes
2196 -----
2197 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2198 This is intended to indicate pixels that may be affected by unknown systematics;
2199 for example if non-linearity corrections above a certain level are unstable
2200 then that would be a useful value for suspectLevel. A value of `nan` indicates
2201 that no such level exists and no pixels are to be masked as suspicious.
2202 """
2203 suspectLevel = amp.getSuspectLevel()
2204 if math.isnan(suspectLevel):
2205 return
2207 maskedImage = exposure.getMaskedImage()
2208 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2209 isrFunctions.makeThresholdMask(
2210 maskedImage=dataView,
2211 threshold=suspectLevel,
2212 growFootprints=0,
2213 maskName=self.config.suspectMaskName,
2214 )
2216 def maskDefect(self, exposure, defectBaseList):
2217 """!Mask defects using mask plane "BAD", in place.
2219 Parameters
2220 ----------
2221 exposure : `lsst.afw.image.Exposure`
2222 Exposure to process.
2223 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2224 `lsst.afw.image.DefectBase`.
2225 List of defects to mask.
2227 Notes
2228 -----
2229 Call this after CCD assembly, since defects may cross amplifier boundaries.
2230 """
2231 maskedImage = exposure.getMaskedImage()
2232 if not isinstance(defectBaseList, Defects):
2233 # Promotes DefectBase to Defect
2234 defectList = Defects(defectBaseList)
2235 else:
2236 defectList = defectBaseList
2237 defectList.maskPixels(maskedImage, maskName="BAD")
2239 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"):
2240 """!Mask edge pixels with applicable mask plane.
2242 Parameters
2243 ----------
2244 exposure : `lsst.afw.image.Exposure`
2245 Exposure to process.
2246 numEdgePixels : `int`, optional
2247 Number of edge pixels to mask.
2248 maskPlane : `str`, optional
2249 Mask plane name to use.
2250 """
2251 maskedImage = exposure.getMaskedImage()
2252 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2254 if numEdgePixels > 0:
2255 goodBBox = maskedImage.getBBox()
2256 # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2257 goodBBox.grow(-numEdgePixels)
2258 # Mask pixels outside goodBBox
2259 SourceDetectionTask.setEdgeBits(
2260 maskedImage,
2261 goodBBox,
2262 maskBitMask
2263 )
2265 def maskAndInterpolateDefects(self, exposure, defectBaseList):
2266 """Mask and interpolate defects using mask plane "BAD", in place.
2268 Parameters
2269 ----------
2270 exposure : `lsst.afw.image.Exposure`
2271 Exposure to process.
2272 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2273 `lsst.afw.image.DefectBase`.
2274 List of defects to mask and interpolate.
2276 See Also
2277 --------
2278 lsst.ip.isr.isrTask.maskDefect()
2279 """
2280 self.maskDefect(exposure, defectBaseList)
2281 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2282 maskPlane="SUSPECT")
2283 isrFunctions.interpolateFromMask(
2284 maskedImage=exposure.getMaskedImage(),
2285 fwhm=self.config.fwhm,
2286 growSaturatedFootprints=0,
2287 maskNameList=["BAD"],
2288 )
2290 def maskNan(self, exposure):
2291 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2293 Parameters
2294 ----------
2295 exposure : `lsst.afw.image.Exposure`
2296 Exposure to process.
2298 Notes
2299 -----
2300 We mask over all NaNs, including those that are masked with
2301 other bits (because those may or may not be interpolated over
2302 later, and we want to remove all NaNs). Despite this
2303 behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2304 the historical name.
2305 """
2306 maskedImage = exposure.getMaskedImage()
2308 # Find and mask NaNs
2309 maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2310 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2311 numNans = maskNans(maskedImage, maskVal)
2312 self.metadata.set("NUMNANS", numNans)
2313 if numNans > 0:
2314 self.log.warn("There were %d unmasked NaNs.", numNans)
2316 def maskAndInterpolateNan(self, exposure):
2317 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2319 Parameters
2320 ----------
2321 exposure : `lsst.afw.image.Exposure`
2322 Exposure to process.
2324 See Also
2325 --------
2326 lsst.ip.isr.isrTask.maskNan()
2327 """
2328 self.maskNan(exposure)
2329 isrFunctions.interpolateFromMask(
2330 maskedImage=exposure.getMaskedImage(),
2331 fwhm=self.config.fwhm,
2332 growSaturatedFootprints=0,
2333 maskNameList=["UNMASKEDNAN"],
2334 )
2336 def measureBackground(self, exposure, IsrQaConfig=None):
2337 """Measure the image background in subgrids, for quality control purposes.
2339 Parameters
2340 ----------
2341 exposure : `lsst.afw.image.Exposure`
2342 Exposure to process.
2343 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2344 Configuration object containing parameters on which background
2345 statistics and subgrids to use.
2346 """
2347 if IsrQaConfig is not None:
2348 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2349 IsrQaConfig.flatness.nIter)
2350 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2351 statsControl.setAndMask(maskVal)
2352 maskedImage = exposure.getMaskedImage()
2353 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2354 skyLevel = stats.getValue(afwMath.MEDIAN)
2355 skySigma = stats.getValue(afwMath.STDEVCLIP)
2356 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2357 metadata = exposure.getMetadata()
2358 metadata.set('SKYLEVEL', skyLevel)
2359 metadata.set('SKYSIGMA', skySigma)
2361 # calcluating flatlevel over the subgrids
2362 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2363 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2364 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2365 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2366 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2367 skyLevels = numpy.zeros((nX, nY))
2369 for j in range(nY):
2370 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2371 for i in range(nX):
2372 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2374 xLLC = xc - meshXHalf
2375 yLLC = yc - meshYHalf
2376 xURC = xc + meshXHalf - 1
2377 yURC = yc + meshYHalf - 1
2379 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2380 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2382 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2384 good = numpy.where(numpy.isfinite(skyLevels))
2385 skyMedian = numpy.median(skyLevels[good])
2386 flatness = (skyLevels[good] - skyMedian) / skyMedian
2387 flatness_rms = numpy.std(flatness)
2388 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2390 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2391 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2392 nX, nY, flatness_pp, flatness_rms)
2394 metadata.set('FLATNESS_PP', float(flatness_pp))
2395 metadata.set('FLATNESS_RMS', float(flatness_rms))
2396 metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2397 metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2398 metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2400 def roughZeroPoint(self, exposure):
2401 """Set an approximate magnitude zero point for the exposure.
2403 Parameters
2404 ----------
2405 exposure : `lsst.afw.image.Exposure`
2406 Exposure to process.
2407 """
2408 filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2409 if filterName in self.config.fluxMag0T1:
2410 fluxMag0 = self.config.fluxMag0T1[filterName]
2411 else:
2412 self.log.warn("No rough magnitude zero point set for filter %s.", filterName)
2413 fluxMag0 = self.config.defaultFluxMag0T1
2415 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2416 if not expTime > 0: # handle NaN as well as <= 0
2417 self.log.warn("Non-positive exposure time; skipping rough zero point.")
2418 return
2420 self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2421 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2423 def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2424 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2426 Parameters
2427 ----------
2428 ccdExposure : `lsst.afw.image.Exposure`
2429 Exposure to process.
2430 fpPolygon : `lsst.afw.geom.Polygon`
2431 Polygon in focal plane coordinates.
2432 """
2433 # Get ccd corners in focal plane coordinates
2434 ccd = ccdExposure.getDetector()
2435 fpCorners = ccd.getCorners(FOCAL_PLANE)
2436 ccdPolygon = Polygon(fpCorners)
2438 # Get intersection of ccd corners with fpPolygon
2439 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2441 # Transform back to pixel positions and build new polygon
2442 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2443 validPolygon = Polygon(ccdPoints)
2444 ccdExposure.getInfo().setValidPolygon(validPolygon)
2446 @contextmanager
2447 def flatContext(self, exp, flat, dark=None):
2448 """Context manager that applies and removes flats and darks,
2449 if the task is configured to apply them.
2451 Parameters
2452 ----------
2453 exp : `lsst.afw.image.Exposure`
2454 Exposure to process.
2455 flat : `lsst.afw.image.Exposure`
2456 Flat exposure the same size as ``exp``.
2457 dark : `lsst.afw.image.Exposure`, optional
2458 Dark exposure the same size as ``exp``.
2460 Yields
2461 ------
2462 exp : `lsst.afw.image.Exposure`
2463 The flat and dark corrected exposure.
2464 """
2465 if self.config.doDark and dark is not None:
2466 self.darkCorrection(exp, dark)
2467 if self.config.doFlat:
2468 self.flatCorrection(exp, flat)
2469 try:
2470 yield exp
2471 finally:
2472 if self.config.doFlat:
2473 self.flatCorrection(exp, flat, invert=True)
2474 if self.config.doDark and dark is not None:
2475 self.darkCorrection(exp, dark, invert=True)
2477 def debugView(self, exposure, stepname):
2478 """Utility function to examine ISR exposure at different stages.
2480 Parameters
2481 ----------
2482 exposure : `lsst.afw.image.Exposure`
2483 Exposure to view.
2484 stepname : `str`
2485 State of processing to view.
2486 """
2487 frame = getDebugFrame(self._display, stepname)
2488 if frame:
2489 display = getDisplay(frame)
2490 display.scale('asinh', 'zscale')
2491 display.mtv(exposure)
2492 prompt = "Press Enter to continue [c]... "
2493 while True:
2494 ans = input(prompt).lower()
2495 if ans in ("", "c",):
2496 break
2499class FakeAmp(object):
2500 """A Detector-like object that supports returning gain and saturation level
2502 This is used when the input exposure does not have a detector.
2504 Parameters
2505 ----------
2506 exposure : `lsst.afw.image.Exposure`
2507 Exposure to generate a fake amplifier for.
2508 config : `lsst.ip.isr.isrTaskConfig`
2509 Configuration to apply to the fake amplifier.
2510 """
2512 def __init__(self, exposure, config):
2513 self._bbox = exposure.getBBox(afwImage.LOCAL)
2514 self._RawHorizontalOverscanBBox = lsst.geom.Box2I()
2515 self._gain = config.gain
2516 self._readNoise = config.readNoise
2517 self._saturation = config.saturation
2519 def getBBox(self):
2520 return self._bbox
2522 def getRawBBox(self):
2523 return self._bbox
2525 def getRawHorizontalOverscanBBox(self):
2526 return self._RawHorizontalOverscanBBox
2528 def getGain(self):
2529 return self._gain
2531 def getReadNoise(self):
2532 return self._readNoise
2534 def getSaturation(self):
2535 return self._saturation
2537 def getSuspectLevel(self):
2538 return float("NaN")
2541class RunIsrConfig(pexConfig.Config):
2542 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2545class RunIsrTask(pipeBase.CmdLineTask):
2546 """Task to wrap the default IsrTask to allow it to be retargeted.
2548 The standard IsrTask can be called directly from a command line
2549 program, but doing so removes the ability of the task to be
2550 retargeted. As most cameras override some set of the IsrTask
2551 methods, this would remove those data-specific methods in the
2552 output post-ISR images. This wrapping class fixes the issue,
2553 allowing identical post-ISR images to be generated by both the
2554 processCcd and isrTask code.
2555 """
2556 ConfigClass = RunIsrConfig
2557 _DefaultName = "runIsr"
2559 def __init__(self, *args, **kwargs):
2560 super().__init__(*args, **kwargs)
2561 self.makeSubtask("isr")
2563 def runDataRef(self, dataRef):
2564 """
2565 Parameters
2566 ----------
2567 dataRef : `lsst.daf.persistence.ButlerDataRef`
2568 data reference of the detector data to be processed
2570 Returns
2571 -------
2572 result : `pipeBase.Struct`
2573 Result struct with component:
2575 - exposure : `lsst.afw.image.Exposure`
2576 Post-ISR processed exposure.
2577 """
2578 return self.isr.runDataRef(dataRef)