Coverage for python/lsst/ip/isr/isrTask.py : 17%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import math
23import numpy
25import lsst.geom
26import lsst.afw.image as afwImage
27import lsst.afw.math as afwMath
28import lsst.pex.config as pexConfig
29import lsst.pipe.base as pipeBase
30import lsst.pipe.base.connectionTypes as cT
32from contextlib import contextmanager
33from lsstDebug import getDebugFrame
35from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36 ReadoutCorner)
37from lsst.afw.display import getDisplay
38from lsst.afw.geom import Polygon
39from lsst.daf.persistence import ButlerDataRef
40from lsst.daf.persistence.butler import NoResults
41from lsst.meas.algorithms.detection import SourceDetectionTask
42from lsst.meas.algorithms import Defects
44from . import isrFunctions
45from . import isrQa
46from . import linearize
48from .assembleCcdTask import AssembleCcdTask
49from .crosstalk import CrosstalkTask, CrosstalkCalib
50from .fringe import FringeTask
51from .isr import maskNans
52from .masking import MaskingTask
53from .overscan import OverscanCorrectionTask
54from .straylight import StrayLightTask
55from .vignette import VignetteTask
56from lsst.daf.butler import DataCoordinate, DimensionGraph, DimensionUniverse
59__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
62def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections):
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
67 populated.
69 This will be unused until DM-25348 resolves the quantum graph
70 generation issue.
72 Parameters
73 ----------
74 datasetType : `str`
75 Dataset to lookup.
76 registry : `lsst.daf.butler.Registry`
77 Butler registry to query.
78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79 Data id to transform to identify crosstalkSources. The
80 ``detector`` entry will be stripped.
81 collections : `lsst.daf.butler.CollectionSearch`
82 Collections to search through.
84 Returns
85 -------
86 results : `list` [`lsst.afw.image.Exposure`]
87 List of datasets that match the query that will be used as
88 crosstalkSources.
89 """
90 newDataId = DataCoordinate(DimensionGraph(DimensionUniverse(),
91 names=('instrument', 'exposure')),
92 (quantumDataId['instrument'], quantumDataId['exposure']))
93 results = list(registry.queryDatasets(datasetType,
94 collections=collections,
95 dataId=newDataId,
96 deduplicate=True,
97 expand=True))
98 return(results)
101class IsrTaskConnections(pipeBase.PipelineTaskConnections,
102 dimensions={"instrument", "exposure", "detector"},
103 defaultTemplates={}):
104 ccdExposure = cT.Input(
105 name="raw",
106 doc="Input exposure to process.",
107 storageClass="Exposure",
108 dimensions=["instrument", "exposure", "detector"],
109 )
110 camera = cT.PrerequisiteInput(
111 name="camera",
112 storageClass="Camera",
113 doc="Input camera to construct complete exposures.",
114 dimensions=["instrument", "calibration_label"],
115 )
117 crosstalk = cT.PrerequisiteInput(
118 name="crosstalk",
119 doc="Input crosstalk object",
120 storageClass="CrosstalkCalib",
121 dimensions=["instrument", "calibration_label", "detector"],
122 )
123 # TODO: DM-25348. This does not work yet to correctly load
124 # possible crosstalk sources.
125 crosstalkSources = cT.PrerequisiteInput(
126 name="isrOverscanCorrected",
127 doc="Overscan corrected input images.",
128 storageClass="Exposure",
129 dimensions=["instrument", "exposure", "detector"],
130 deferLoad=True,
131 multiple=True,
132 lookupFunction=crosstalkSourceLookup,
133 )
134 bias = cT.PrerequisiteInput(
135 name="bias",
136 doc="Input bias calibration.",
137 storageClass="ExposureF",
138 dimensions=["instrument", "calibration_label", "detector"],
139 )
140 dark = cT.PrerequisiteInput(
141 name='dark',
142 doc="Input dark calibration.",
143 storageClass="ExposureF",
144 dimensions=["instrument", "calibration_label", "detector"],
145 )
146 flat = cT.PrerequisiteInput(
147 name="flat",
148 doc="Input flat calibration.",
149 storageClass="ExposureF",
150 dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
151 )
152 fringes = cT.PrerequisiteInput(
153 name="fringe",
154 doc="Input fringe calibration.",
155 storageClass="ExposureF",
156 dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
157 )
158 strayLightData = cT.PrerequisiteInput(
159 name='yBackground',
160 doc="Input stray light calibration.",
161 storageClass="StrayLightData",
162 dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
163 )
164 bfKernel = cT.PrerequisiteInput(
165 name='bfKernel',
166 doc="Input brighter-fatter kernel.",
167 storageClass="NumpyArray",
168 dimensions=["instrument", "calibration_label"],
169 )
170 newBFKernel = cT.PrerequisiteInput(
171 name='brighterFatterKernel',
172 doc="Newer complete kernel + gain solutions.",
173 storageClass="BrighterFatterKernel",
174 dimensions=["instrument", "calibration_label", "detector"],
175 )
176 defects = cT.PrerequisiteInput(
177 name='defects',
178 doc="Input defect tables.",
179 storageClass="Defects",
180 dimensions=["instrument", "calibration_label", "detector"],
181 )
182 opticsTransmission = cT.PrerequisiteInput(
183 name="transmission_optics",
184 storageClass="TransmissionCurve",
185 doc="Transmission curve due to the optics.",
186 dimensions=["instrument", "calibration_label"],
187 )
188 filterTransmission = cT.PrerequisiteInput(
189 name="transmission_filter",
190 storageClass="TransmissionCurve",
191 doc="Transmission curve due to the filter.",
192 dimensions=["instrument", "physical_filter", "calibration_label"],
193 )
194 sensorTransmission = cT.PrerequisiteInput(
195 name="transmission_sensor",
196 storageClass="TransmissionCurve",
197 doc="Transmission curve due to the sensor.",
198 dimensions=["instrument", "calibration_label", "detector"],
199 )
200 atmosphereTransmission = cT.PrerequisiteInput(
201 name="transmission_atmosphere",
202 storageClass="TransmissionCurve",
203 doc="Transmission curve due to the atmosphere.",
204 dimensions=["instrument"],
205 )
206 illumMaskedImage = cT.PrerequisiteInput(
207 name="illum",
208 doc="Input illumination correction.",
209 storageClass="MaskedImageF",
210 dimensions=["instrument", "physical_filter", "calibration_label", "detector"],
211 )
213 outputExposure = cT.Output(
214 name='postISRCCD',
215 doc="Output ISR processed exposure.",
216 storageClass="Exposure",
217 dimensions=["instrument", "exposure", "detector"],
218 )
219 preInterpExposure = cT.Output(
220 name='preInterpISRCCD',
221 doc="Output ISR processed exposure, with pixels left uninterpolated.",
222 storageClass="ExposureF",
223 dimensions=["instrument", "exposure", "detector"],
224 )
225 outputOssThumbnail = cT.Output(
226 name="OssThumb",
227 doc="Output Overscan-subtracted thumbnail image.",
228 storageClass="Thumbnail",
229 dimensions=["instrument", "exposure", "detector"],
230 )
231 outputFlattenedThumbnail = cT.Output(
232 name="FlattenedThumb",
233 doc="Output flat-corrected thumbnail image.",
234 storageClass="Thumbnail",
235 dimensions=["instrument", "exposure", "detector"],
236 )
238 def __init__(self, *, config=None):
239 super().__init__(config=config)
241 if config.doBias is not True:
242 self.prerequisiteInputs.discard("bias")
243 if config.doLinearize is not True:
244 self.prerequisiteInputs.discard("linearizer")
245 if config.doCrosstalk is not True:
246 self.inputs.discard("crosstalkSources")
247 self.prerequisiteInputs.discard("crosstalk")
248 if config.doBrighterFatter is not True:
249 self.prerequisiteInputs.discard("bfKernel")
250 self.prerequisiteInputs.discard("newBFKernel")
251 if config.doDefect is not True:
252 self.prerequisiteInputs.discard("defects")
253 if config.doDark is not True:
254 self.prerequisiteInputs.discard("dark")
255 if config.doFlat is not True:
256 self.prerequisiteInputs.discard("flat")
257 if config.doAttachTransmissionCurve is not True:
258 self.prerequisiteInputs.discard("opticsTransmission")
259 self.prerequisiteInputs.discard("filterTransmission")
260 self.prerequisiteInputs.discard("sensorTransmission")
261 self.prerequisiteInputs.discard("atmosphereTransmission")
262 if config.doUseOpticsTransmission is not True:
263 self.prerequisiteInputs.discard("opticsTransmission")
264 if config.doUseFilterTransmission is not True:
265 self.prerequisiteInputs.discard("filterTransmission")
266 if config.doUseSensorTransmission is not True:
267 self.prerequisiteInputs.discard("sensorTransmission")
268 if config.doUseAtmosphereTransmission is not True:
269 self.prerequisiteInputs.discard("atmosphereTransmission")
270 if config.doIlluminationCorrection is not True:
271 self.prerequisiteInputs.discard("illumMaskedImage")
273 if config.doWrite is not True:
274 self.outputs.discard("outputExposure")
275 self.outputs.discard("preInterpExposure")
276 self.outputs.discard("outputFlattenedThumbnail")
277 self.outputs.discard("outputOssThumbnail")
278 if config.doSaveInterpPixels is not True:
279 self.outputs.discard("preInterpExposure")
280 if config.qa.doThumbnailOss is not True:
281 self.outputs.discard("outputOssThumbnail")
282 if config.qa.doThumbnailFlattened is not True:
283 self.outputs.discard("outputFlattenedThumbnail")
286class IsrTaskConfig(pipeBase.PipelineTaskConfig,
287 pipelineConnections=IsrTaskConnections):
288 """Configuration parameters for IsrTask.
290 Items are grouped in the order in which they are executed by the task.
291 """
292 datasetType = pexConfig.Field(
293 dtype=str,
294 doc="Dataset type for input data; users will typically leave this alone, "
295 "but camera-specific ISR tasks will override it",
296 default="raw",
297 )
299 fallbackFilterName = pexConfig.Field(
300 dtype=str,
301 doc="Fallback default filter name for calibrations.",
302 optional=True
303 )
304 useFallbackDate = pexConfig.Field(
305 dtype=bool,
306 doc="Pass observation date when using fallback filter.",
307 default=False,
308 )
309 expectWcs = pexConfig.Field(
310 dtype=bool,
311 default=True,
312 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
313 )
314 fwhm = pexConfig.Field(
315 dtype=float,
316 doc="FWHM of PSF in arcseconds.",
317 default=1.0,
318 )
319 qa = pexConfig.ConfigField(
320 dtype=isrQa.IsrQaConfig,
321 doc="QA related configuration options.",
322 )
324 # Image conversion configuration
325 doConvertIntToFloat = pexConfig.Field(
326 dtype=bool,
327 doc="Convert integer raw images to floating point values?",
328 default=True,
329 )
331 # Saturated pixel handling.
332 doSaturation = pexConfig.Field(
333 dtype=bool,
334 doc="Mask saturated pixels? NB: this is totally independent of the"
335 " interpolation option - this is ONLY setting the bits in the mask."
336 " To have them interpolated make sure doSaturationInterpolation=True",
337 default=True,
338 )
339 saturatedMaskName = pexConfig.Field(
340 dtype=str,
341 doc="Name of mask plane to use in saturation detection and interpolation",
342 default="SAT",
343 )
344 saturation = pexConfig.Field(
345 dtype=float,
346 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
347 default=float("NaN"),
348 )
349 growSaturationFootprintSize = pexConfig.Field(
350 dtype=int,
351 doc="Number of pixels by which to grow the saturation footprints",
352 default=1,
353 )
355 # Suspect pixel handling.
356 doSuspect = pexConfig.Field(
357 dtype=bool,
358 doc="Mask suspect pixels?",
359 default=False,
360 )
361 suspectMaskName = pexConfig.Field(
362 dtype=str,
363 doc="Name of mask plane to use for suspect pixels",
364 default="SUSPECT",
365 )
366 numEdgeSuspect = pexConfig.Field(
367 dtype=int,
368 doc="Number of edge pixels to be flagged as untrustworthy.",
369 default=0,
370 )
372 # Initial masking options.
373 doSetBadRegions = pexConfig.Field(
374 dtype=bool,
375 doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
376 default=True,
377 )
378 badStatistic = pexConfig.ChoiceField(
379 dtype=str,
380 doc="How to estimate the average value for BAD regions.",
381 default='MEANCLIP',
382 allowed={
383 "MEANCLIP": "Correct using the (clipped) mean of good data",
384 "MEDIAN": "Correct using the median of the good data",
385 },
386 )
388 # Overscan subtraction configuration.
389 doOverscan = pexConfig.Field(
390 dtype=bool,
391 doc="Do overscan subtraction?",
392 default=True,
393 )
394 overscan = pexConfig.ConfigurableField(
395 target=OverscanCorrectionTask,
396 doc="Overscan subtraction task for image segments.",
397 )
399 overscanFitType = pexConfig.ChoiceField(
400 dtype=str,
401 doc="The method for fitting the overscan bias level.",
402 default='MEDIAN',
403 allowed={
404 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
405 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
406 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
407 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
408 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
409 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
410 "MEAN": "Correct using the mean of the overscan region",
411 "MEANCLIP": "Correct using a clipped mean of the overscan region",
412 "MEDIAN": "Correct using the median of the overscan region",
413 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
414 },
415 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
416 " This option will no longer be used, and will be removed after v20.")
417 )
418 overscanOrder = pexConfig.Field(
419 dtype=int,
420 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, "
421 "or number of spline knots if overscan fit type is a spline."),
422 default=1,
423 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
424 " This option will no longer be used, and will be removed after v20.")
425 )
426 overscanNumSigmaClip = pexConfig.Field(
427 dtype=float,
428 doc="Rejection threshold (sigma) for collapsing overscan before fit",
429 default=3.0,
430 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
431 " This option will no longer be used, and will be removed after v20.")
432 )
433 overscanIsInt = pexConfig.Field(
434 dtype=bool,
435 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
436 " and overscan.FitType=MEDIAN_PER_ROW.",
437 default=True,
438 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
439 " This option will no longer be used, and will be removed after v20.")
440 )
441 # These options do not get deprecated, as they define how we slice up the image data.
442 overscanNumLeadingColumnsToSkip = pexConfig.Field(
443 dtype=int,
444 doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
445 default=0,
446 )
447 overscanNumTrailingColumnsToSkip = pexConfig.Field(
448 dtype=int,
449 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
450 default=0,
451 )
452 overscanMaxDev = pexConfig.Field( 452 ↛ exitline 452 didn't jump to the function exit
453 dtype=float,
454 doc="Maximum deviation from the median for overscan",
455 default=1000.0, check=lambda x: x > 0
456 )
457 overscanBiasJump = pexConfig.Field(
458 dtype=bool,
459 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
460 default=False,
461 )
462 overscanBiasJumpKeyword = pexConfig.Field(
463 dtype=str,
464 doc="Header keyword containing information about devices.",
465 default="NO_SUCH_KEY",
466 )
467 overscanBiasJumpDevices = pexConfig.ListField(
468 dtype=str,
469 doc="List of devices that need piecewise overscan correction.",
470 default=(),
471 )
472 overscanBiasJumpLocation = pexConfig.Field(
473 dtype=int,
474 doc="Location of bias jump along y-axis.",
475 default=0,
476 )
478 # Amplifier to CCD assembly configuration
479 doAssembleCcd = pexConfig.Field(
480 dtype=bool,
481 default=True,
482 doc="Assemble amp-level exposures into a ccd-level exposure?"
483 )
484 assembleCcd = pexConfig.ConfigurableField(
485 target=AssembleCcdTask,
486 doc="CCD assembly task",
487 )
489 # General calibration configuration.
490 doAssembleIsrExposures = pexConfig.Field(
491 dtype=bool,
492 default=False,
493 doc="Assemble amp-level calibration exposures into ccd-level exposure?"
494 )
495 doTrimToMatchCalib = pexConfig.Field(
496 dtype=bool,
497 default=False,
498 doc="Trim raw data to match calibration bounding boxes?"
499 )
501 # Bias subtraction.
502 doBias = pexConfig.Field(
503 dtype=bool,
504 doc="Apply bias frame correction?",
505 default=True,
506 )
507 biasDataProductName = pexConfig.Field(
508 dtype=str,
509 doc="Name of the bias data product",
510 default="bias",
511 )
513 # Variance construction
514 doVariance = pexConfig.Field(
515 dtype=bool,
516 doc="Calculate variance?",
517 default=True
518 )
519 gain = pexConfig.Field(
520 dtype=float,
521 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
522 default=float("NaN"),
523 )
524 readNoise = pexConfig.Field(
525 dtype=float,
526 doc="The read noise to use if no Detector is present in the Exposure",
527 default=0.0,
528 )
529 doEmpiricalReadNoise = pexConfig.Field(
530 dtype=bool,
531 default=False,
532 doc="Calculate empirical read noise instead of value from AmpInfo data?"
533 )
535 # Linearization.
536 doLinearize = pexConfig.Field(
537 dtype=bool,
538 doc="Correct for nonlinearity of the detector's response?",
539 default=True,
540 )
542 # Crosstalk.
543 doCrosstalk = pexConfig.Field(
544 dtype=bool,
545 doc="Apply intra-CCD crosstalk correction?",
546 default=False,
547 )
548 doCrosstalkBeforeAssemble = pexConfig.Field(
549 dtype=bool,
550 doc="Apply crosstalk correction before CCD assembly, and before trimming?",
551 default=False,
552 )
553 crosstalk = pexConfig.ConfigurableField(
554 target=CrosstalkTask,
555 doc="Intra-CCD crosstalk correction",
556 )
558 # Masking options.
559 doDefect = pexConfig.Field(
560 dtype=bool,
561 doc="Apply correction for CCD defects, e.g. hot pixels?",
562 default=True,
563 )
564 doNanMasking = pexConfig.Field(
565 dtype=bool,
566 doc="Mask NAN pixels?",
567 default=True,
568 )
569 doWidenSaturationTrails = pexConfig.Field(
570 dtype=bool,
571 doc="Widen bleed trails based on their width?",
572 default=True
573 )
575 # Brighter-Fatter correction.
576 doBrighterFatter = pexConfig.Field(
577 dtype=bool,
578 default=False,
579 doc="Apply the brighter fatter correction"
580 )
581 brighterFatterLevel = pexConfig.ChoiceField(
582 dtype=str,
583 default="DETECTOR",
584 doc="The level at which to correct for brighter-fatter.",
585 allowed={
586 "AMP": "Every amplifier treated separately.",
587 "DETECTOR": "One kernel per detector",
588 }
589 )
590 brighterFatterMaxIter = pexConfig.Field(
591 dtype=int,
592 default=10,
593 doc="Maximum number of iterations for the brighter fatter correction"
594 )
595 brighterFatterThreshold = pexConfig.Field(
596 dtype=float,
597 default=1000,
598 doc="Threshold used to stop iterating the brighter fatter correction. It is the "
599 " absolute value of the difference between the current corrected image and the one"
600 " from the previous iteration summed over all the pixels."
601 )
602 brighterFatterApplyGain = pexConfig.Field(
603 dtype=bool,
604 default=True,
605 doc="Should the gain be applied when applying the brighter fatter correction?"
606 )
607 brighterFatterMaskGrowSize = pexConfig.Field(
608 dtype=int,
609 default=0,
610 doc="Number of pixels to grow the masks listed in config.maskListToInterpolate "
611 " when brighter-fatter correction is applied."
612 )
614 # Dark subtraction.
615 doDark = pexConfig.Field(
616 dtype=bool,
617 doc="Apply dark frame correction?",
618 default=True,
619 )
620 darkDataProductName = pexConfig.Field(
621 dtype=str,
622 doc="Name of the dark data product",
623 default="dark",
624 )
626 # Camera-specific stray light removal.
627 doStrayLight = pexConfig.Field(
628 dtype=bool,
629 doc="Subtract stray light in the y-band (due to encoder LEDs)?",
630 default=False,
631 )
632 strayLight = pexConfig.ConfigurableField(
633 target=StrayLightTask,
634 doc="y-band stray light correction"
635 )
637 # Flat correction.
638 doFlat = pexConfig.Field(
639 dtype=bool,
640 doc="Apply flat field correction?",
641 default=True,
642 )
643 flatDataProductName = pexConfig.Field(
644 dtype=str,
645 doc="Name of the flat data product",
646 default="flat",
647 )
648 flatScalingType = pexConfig.ChoiceField(
649 dtype=str,
650 doc="The method for scaling the flat on the fly.",
651 default='USER',
652 allowed={
653 "USER": "Scale by flatUserScale",
654 "MEAN": "Scale by the inverse of the mean",
655 "MEDIAN": "Scale by the inverse of the median",
656 },
657 )
658 flatUserScale = pexConfig.Field(
659 dtype=float,
660 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
661 default=1.0,
662 )
663 doTweakFlat = pexConfig.Field(
664 dtype=bool,
665 doc="Tweak flats to match observed amplifier ratios?",
666 default=False
667 )
669 # Amplifier normalization based on gains instead of using flats configuration.
670 doApplyGains = pexConfig.Field(
671 dtype=bool,
672 doc="Correct the amplifiers for their gains instead of applying flat correction",
673 default=False,
674 )
675 normalizeGains = pexConfig.Field(
676 dtype=bool,
677 doc="Normalize all the amplifiers in each CCD to have the same median value.",
678 default=False,
679 )
681 # Fringe correction.
682 doFringe = pexConfig.Field(
683 dtype=bool,
684 doc="Apply fringe correction?",
685 default=True,
686 )
687 fringe = pexConfig.ConfigurableField(
688 target=FringeTask,
689 doc="Fringe subtraction task",
690 )
691 fringeAfterFlat = pexConfig.Field(
692 dtype=bool,
693 doc="Do fringe subtraction after flat-fielding?",
694 default=True,
695 )
697 # Initial CCD-level background statistics options.
698 doMeasureBackground = pexConfig.Field(
699 dtype=bool,
700 doc="Measure the background level on the reduced image?",
701 default=False,
702 )
704 # Camera-specific masking configuration.
705 doCameraSpecificMasking = pexConfig.Field(
706 dtype=bool,
707 doc="Mask camera-specific bad regions?",
708 default=False,
709 )
710 masking = pexConfig.ConfigurableField(
711 target=MaskingTask,
712 doc="Masking task."
713 )
715 # Interpolation options.
717 doInterpolate = pexConfig.Field(
718 dtype=bool,
719 doc="Interpolate masked pixels?",
720 default=True,
721 )
722 doSaturationInterpolation = pexConfig.Field(
723 dtype=bool,
724 doc="Perform interpolation over pixels masked as saturated?"
725 " NB: This is independent of doSaturation; if that is False this plane"
726 " will likely be blank, resulting in a no-op here.",
727 default=True,
728 )
729 doNanInterpolation = pexConfig.Field(
730 dtype=bool,
731 doc="Perform interpolation over pixels masked as NaN?"
732 " NB: This is independent of doNanMasking; if that is False this plane"
733 " will likely be blank, resulting in a no-op here.",
734 default=True,
735 )
736 doNanInterpAfterFlat = pexConfig.Field(
737 dtype=bool,
738 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
739 "also have to interpolate them before flat-fielding."),
740 default=False,
741 )
742 maskListToInterpolate = pexConfig.ListField(
743 dtype=str,
744 doc="List of mask planes that should be interpolated.",
745 default=['SAT', 'BAD', 'UNMASKEDNAN'],
746 )
747 doSaveInterpPixels = pexConfig.Field(
748 dtype=bool,
749 doc="Save a copy of the pre-interpolated pixel values?",
750 default=False,
751 )
753 # Default photometric calibration options.
754 fluxMag0T1 = pexConfig.DictField(
755 keytype=str,
756 itemtype=float,
757 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
758 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
759 ))
760 )
761 defaultFluxMag0T1 = pexConfig.Field(
762 dtype=float,
763 doc="Default value for fluxMag0T1 (for an unrecognized filter).",
764 default=pow(10.0, 0.4*28.0)
765 )
767 # Vignette correction configuration.
768 doVignette = pexConfig.Field(
769 dtype=bool,
770 doc="Apply vignetting parameters?",
771 default=False,
772 )
773 vignette = pexConfig.ConfigurableField(
774 target=VignetteTask,
775 doc="Vignetting task.",
776 )
778 # Transmission curve configuration.
779 doAttachTransmissionCurve = pexConfig.Field(
780 dtype=bool,
781 default=False,
782 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
783 )
784 doUseOpticsTransmission = pexConfig.Field(
785 dtype=bool,
786 default=True,
787 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
788 )
789 doUseFilterTransmission = pexConfig.Field(
790 dtype=bool,
791 default=True,
792 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
793 )
794 doUseSensorTransmission = pexConfig.Field(
795 dtype=bool,
796 default=True,
797 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
798 )
799 doUseAtmosphereTransmission = pexConfig.Field(
800 dtype=bool,
801 default=True,
802 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
803 )
805 # Illumination correction.
806 doIlluminationCorrection = pexConfig.Field(
807 dtype=bool,
808 default=False,
809 doc="Perform illumination correction?"
810 )
811 illuminationCorrectionDataProductName = pexConfig.Field(
812 dtype=str,
813 doc="Name of the illumination correction data product.",
814 default="illumcor",
815 )
816 illumScale = pexConfig.Field(
817 dtype=float,
818 doc="Scale factor for the illumination correction.",
819 default=1.0,
820 )
821 illumFilters = pexConfig.ListField(
822 dtype=str,
823 default=[],
824 doc="Only perform illumination correction for these filters."
825 )
827 # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
828 doWrite = pexConfig.Field(
829 dtype=bool,
830 doc="Persist postISRCCD?",
831 default=True,
832 )
834 def validate(self):
835 super().validate()
836 if self.doFlat and self.doApplyGains:
837 raise ValueError("You may not specify both doFlat and doApplyGains")
838 if self.doSaturationInterpolation and "SAT" not in self.maskListToInterpolate:
839 self.config.maskListToInterpolate.append("SAT")
840 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
841 self.config.maskListToInterpolate.append("UNMASKEDNAN")
844class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
845 """Apply common instrument signature correction algorithms to a raw frame.
847 The process for correcting imaging data is very similar from
848 camera to camera. This task provides a vanilla implementation of
849 doing these corrections, including the ability to turn certain
850 corrections off if they are not needed. The inputs to the primary
851 method, `run()`, are a raw exposure to be corrected and the
852 calibration data products. The raw input is a single chip sized
853 mosaic of all amps including overscans and other non-science
854 pixels. The method `runDataRef()` identifies and defines the
855 calibration data products, and is intended for use by a
856 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
857 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
858 subclassed for different camera, although the most camera specific
859 methods have been split into subtasks that can be redirected
860 appropriately.
862 The __init__ method sets up the subtasks for ISR processing, using
863 the defaults from `lsst.ip.isr`.
865 Parameters
866 ----------
867 args : `list`
868 Positional arguments passed to the Task constructor. None used at this time.
869 kwargs : `dict`, optional
870 Keyword arguments passed on to the Task constructor. None used at this time.
871 """
872 ConfigClass = IsrTaskConfig
873 _DefaultName = "isr"
875 def __init__(self, **kwargs):
876 super().__init__(**kwargs)
877 self.makeSubtask("assembleCcd")
878 self.makeSubtask("crosstalk")
879 self.makeSubtask("strayLight")
880 self.makeSubtask("fringe")
881 self.makeSubtask("masking")
882 self.makeSubtask("overscan")
883 self.makeSubtask("vignette")
885 def runQuantum(self, butlerQC, inputRefs, outputRefs):
886 inputs = butlerQC.get(inputRefs)
888 try:
889 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
890 except Exception as e:
891 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
892 (inputRefs, e))
894 inputs['isGen3'] = True
896 detector = inputs['ccdExposure'].getDetector()
898 if self.config.doCrosstalk is True:
899 # Crosstalk sources need to be defined by the pipeline
900 # yaml if they exist.
901 if 'crosstalk' in inputs and inputs['crosstalk'] is not None:
902 if not isinstance(inputs['crosstalk'], CrosstalkCalib):
903 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk'])
904 else:
905 coeffVector = (self.config.crosstalk.crosstalkValues
906 if self.config.crosstalk.useConfigCoefficients else None)
907 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
908 inputs['crosstalk'] = crosstalkCalib
909 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0:
910 if 'crosstalkSources' not in inputs:
911 self.log.warn("No crosstalkSources found for chip with interChip terms!")
913 if self.doLinearize(detector) is True:
914 if 'linearizer' in inputs and isinstance(inputs['linearizer'], dict):
915 linearizer = linearize.Linearizer(detector=detector, log=self.log)
916 linearizer.fromYaml(inputs['linearizer'])
917 else:
918 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None), detector=detector,
919 log=self.log)
920 inputs['linearizer'] = linearizer
922 if self.config.doDefect is True:
923 if "defects" in inputs and inputs['defects'] is not None:
924 # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
925 # masking expects a list of defects defined by their bounding box
926 if not isinstance(inputs["defects"], Defects):
927 inputs["defects"] = Defects.fromTable(inputs["defects"])
929 # Load the correct style of brighter fatter kernel, and repack
930 # the information as a numpy array.
931 if self.config.doBrighterFatter:
932 brighterFatterKernel = inputs.pop('newBFKernel', None)
933 if brighterFatterKernel is None:
934 brighterFatterKernel = inputs.get('bfKernel', None)
936 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
937 detId = detector.getId()
938 inputs['bfGains'] = brighterFatterKernel.gain
939 # If the kernel is not an ndarray, it's the cp_pipe version
940 # so extract the kernel for this detector, or raise an error
941 if self.config.brighterFatterLevel == 'DETECTOR':
942 if brighterFatterKernel.detectorKernel:
943 inputs['bfKernel'] = brighterFatterKernel.detectorKernel[detId]
944 elif brighterFatterKernel.detectorKernelFromAmpKernels:
945 inputs['bfKernel'] = brighterFatterKernel.detectorKernelFromAmpKernels[detId]
946 else:
947 raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
948 else:
949 # TODO DM-15631 for implementing this
950 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
952 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
953 expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
954 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
955 expId=expId,
956 assembler=self.assembleCcd
957 if self.config.doAssembleIsrExposures else None)
958 else:
959 inputs['fringes'] = pipeBase.Struct(fringes=None)
961 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
962 if 'strayLightData' not in inputs:
963 inputs['strayLightData'] = None
965 outputs = self.run(**inputs)
966 butlerQC.put(outputs, outputRefs)
968 def readIsrData(self, dataRef, rawExposure):
969 """!Retrieve necessary frames for instrument signature removal.
971 Pre-fetching all required ISR data products limits the IO
972 required by the ISR. Any conflict between the calibration data
973 available and that needed for ISR is also detected prior to
974 doing processing, allowing it to fail quickly.
976 Parameters
977 ----------
978 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
979 Butler reference of the detector data to be processed
980 rawExposure : `afw.image.Exposure`
981 The raw exposure that will later be corrected with the
982 retrieved calibration data; should not be modified in this
983 method.
985 Returns
986 -------
987 result : `lsst.pipe.base.Struct`
988 Result struct with components (which may be `None`):
989 - ``bias``: bias calibration frame (`afw.image.Exposure`)
990 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
991 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
992 - ``dark``: dark calibration frame (`afw.image.Exposure`)
993 - ``flat``: flat calibration frame (`afw.image.Exposure`)
994 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
995 - ``defects``: list of defects (`lsst.meas.algorithms.Defects`)
996 - ``fringes``: `lsst.pipe.base.Struct` with components:
997 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
998 - ``seed``: random seed derived from the ccdExposureId for random
999 number generator (`uint32`).
1000 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1001 A ``TransmissionCurve`` that represents the throughput of the optics,
1002 to be evaluated in focal-plane coordinates.
1003 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1004 A ``TransmissionCurve`` that represents the throughput of the filter
1005 itself, to be evaluated in focal-plane coordinates.
1006 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1007 A ``TransmissionCurve`` that represents the throughput of the sensor
1008 itself, to be evaluated in post-assembly trimmed detector coordinates.
1009 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1010 A ``TransmissionCurve`` that represents the throughput of the
1011 atmosphere, assumed to be spatially constant.
1012 - ``strayLightData`` : `object`
1013 An opaque object containing calibration information for
1014 stray-light correction. If `None`, no correction will be
1015 performed.
1016 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1018 Raises
1019 ------
1020 NotImplementedError :
1021 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1022 """
1023 try:
1024 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1025 dateObs = dateObs.toPython().isoformat()
1026 except RuntimeError:
1027 self.log.warn("Unable to identify dateObs for rawExposure.")
1028 dateObs = None
1030 ccd = rawExposure.getDetector()
1031 filterName = afwImage.Filter(rawExposure.getFilter().getId()).getName() # Canonical name for filter
1032 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
1033 biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
1034 if self.config.doBias else None)
1035 # immediate=True required for functors and linearizers are functors; see ticket DM-6515
1036 linearizer = (dataRef.get("linearizer", immediate=True)
1037 if self.doLinearize(ccd) else None)
1038 if linearizer is not None and not isinstance(linearizer, numpy.ndarray):
1039 linearizer.log = self.log
1040 if isinstance(linearizer, numpy.ndarray):
1041 linearizer = linearize.Linearizer(table=linearizer, detector=ccd)
1043 crosstalkCalib = None
1044 if self.config.doCrosstalk:
1045 try:
1046 crosstalkCalib = dataRef.get("crosstalk", immediate=True)
1047 except NoResults:
1048 coeffVector = (self.config.crosstalk.crosstalkValues
1049 if self.config.crosstalk.useConfigCoefficients else None)
1050 crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1051 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1052 if self.config.doCrosstalk else None)
1054 darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
1055 if self.config.doDark else None)
1056 flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName,
1057 dateObs=dateObs)
1058 if self.config.doFlat else None)
1060 brighterFatterKernel = None
1061 brighterFatterGains = None
1062 if self.config.doBrighterFatter is True:
1063 try:
1064 # Use the new-style cp_pipe version of the kernel if it exists
1065 # If using a new-style kernel, always use the self-consistent
1066 # gains, i.e. the ones inside the kernel object itself
1067 brighterFatterKernel = dataRef.get("brighterFatterKernel")
1068 brighterFatterGains = brighterFatterKernel.gain
1069 self.log.info("New style bright-fatter kernel (brighterFatterKernel) loaded")
1070 except NoResults:
1071 try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
1072 brighterFatterKernel = dataRef.get("bfKernel")
1073 self.log.info("Old style bright-fatter kernel (np.array) loaded")
1074 except NoResults:
1075 brighterFatterKernel = None
1076 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1077 # If the kernel is not an ndarray, it's the cp_pipe version
1078 # so extract the kernel for this detector, or raise an error
1079 if self.config.brighterFatterLevel == 'DETECTOR':
1080 if brighterFatterKernel.detectorKernel:
1081 brighterFatterKernel = brighterFatterKernel.detectorKernel[ccd.getId()]
1082 elif brighterFatterKernel.detectorKernelFromAmpKernels:
1083 brighterFatterKernel = brighterFatterKernel.detectorKernelFromAmpKernels[ccd.getId()]
1084 else:
1085 raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1086 else:
1087 # TODO DM-15631 for implementing this
1088 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1090 defectList = (dataRef.get("defects")
1091 if self.config.doDefect else None)
1092 fringeStruct = (self.fringe.readFringes(dataRef, assembler=self.assembleCcd
1093 if self.config.doAssembleIsrExposures else None)
1094 if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1095 else pipeBase.Struct(fringes=None))
1097 if self.config.doAttachTransmissionCurve:
1098 opticsTransmission = (dataRef.get("transmission_optics")
1099 if self.config.doUseOpticsTransmission else None)
1100 filterTransmission = (dataRef.get("transmission_filter")
1101 if self.config.doUseFilterTransmission else None)
1102 sensorTransmission = (dataRef.get("transmission_sensor")
1103 if self.config.doUseSensorTransmission else None)
1104 atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1105 if self.config.doUseAtmosphereTransmission else None)
1106 else:
1107 opticsTransmission = None
1108 filterTransmission = None
1109 sensorTransmission = None
1110 atmosphereTransmission = None
1112 if self.config.doStrayLight:
1113 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1114 else:
1115 strayLightData = None
1117 illumMaskedImage = (self.getIsrExposure(dataRef,
1118 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1119 if (self.config.doIlluminationCorrection
1120 and filterName in self.config.illumFilters)
1121 else None)
1123 # Struct should include only kwargs to run()
1124 return pipeBase.Struct(bias=biasExposure,
1125 linearizer=linearizer,
1126 crosstalk=crosstalkCalib,
1127 crosstalkSources=crosstalkSources,
1128 dark=darkExposure,
1129 flat=flatExposure,
1130 bfKernel=brighterFatterKernel,
1131 bfGains=brighterFatterGains,
1132 defects=defectList,
1133 fringes=fringeStruct,
1134 opticsTransmission=opticsTransmission,
1135 filterTransmission=filterTransmission,
1136 sensorTransmission=sensorTransmission,
1137 atmosphereTransmission=atmosphereTransmission,
1138 strayLightData=strayLightData,
1139 illumMaskedImage=illumMaskedImage
1140 )
1142 @pipeBase.timeMethod
1143 def run(self, ccdExposure, camera=None, bias=None, linearizer=None,
1144 crosstalk=None, crosstalkSources=None,
1145 dark=None, flat=None, bfKernel=None, bfGains=None, defects=None,
1146 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1147 sensorTransmission=None, atmosphereTransmission=None,
1148 detectorNum=None, strayLightData=None, illumMaskedImage=None,
1149 isGen3=False,
1150 ):
1151 """!Perform instrument signature removal on an exposure.
1153 Steps included in the ISR processing, in order performed, are:
1154 - saturation and suspect pixel masking
1155 - overscan subtraction
1156 - CCD assembly of individual amplifiers
1157 - bias subtraction
1158 - variance image construction
1159 - linearization of non-linear response
1160 - crosstalk masking
1161 - brighter-fatter correction
1162 - dark subtraction
1163 - fringe correction
1164 - stray light subtraction
1165 - flat correction
1166 - masking of known defects and camera specific features
1167 - vignette calculation
1168 - appending transmission curve and distortion model
1170 Parameters
1171 ----------
1172 ccdExposure : `lsst.afw.image.Exposure`
1173 The raw exposure that is to be run through ISR. The
1174 exposure is modified by this method.
1175 camera : `lsst.afw.cameraGeom.Camera`, optional
1176 The camera geometry for this exposure. Required if ``isGen3`` is
1177 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1178 ``flat`` does not have an associated detector.
1179 bias : `lsst.afw.image.Exposure`, optional
1180 Bias calibration frame.
1181 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1182 Functor for linearization.
1183 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1184 Calibration for crosstalk.
1185 crosstalkSources : `list`, optional
1186 List of possible crosstalk sources.
1187 dark : `lsst.afw.image.Exposure`, optional
1188 Dark calibration frame.
1189 flat : `lsst.afw.image.Exposure`, optional
1190 Flat calibration frame.
1191 bfKernel : `numpy.ndarray`, optional
1192 Brighter-fatter kernel.
1193 bfGains : `dict` of `float`, optional
1194 Gains used to override the detector's nominal gains for the
1195 brighter-fatter correction. A dict keyed by amplifier name for
1196 the detector in question.
1197 defects : `lsst.meas.algorithms.Defects`, optional
1198 List of defects.
1199 fringes : `lsst.pipe.base.Struct`, optional
1200 Struct containing the fringe correction data, with
1201 elements:
1202 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1203 - ``seed``: random seed derived from the ccdExposureId for random
1204 number generator (`uint32`)
1205 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1206 A ``TransmissionCurve`` that represents the throughput of the optics,
1207 to be evaluated in focal-plane coordinates.
1208 filterTransmission : `lsst.afw.image.TransmissionCurve`
1209 A ``TransmissionCurve`` that represents the throughput of the filter
1210 itself, to be evaluated in focal-plane coordinates.
1211 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1212 A ``TransmissionCurve`` that represents the throughput of the sensor
1213 itself, to be evaluated in post-assembly trimmed detector coordinates.
1214 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1215 A ``TransmissionCurve`` that represents the throughput of the
1216 atmosphere, assumed to be spatially constant.
1217 detectorNum : `int`, optional
1218 The integer number for the detector to process.
1219 isGen3 : bool, optional
1220 Flag this call to run() as using the Gen3 butler environment.
1221 strayLightData : `object`, optional
1222 Opaque object containing calibration information for stray-light
1223 correction. If `None`, no correction will be performed.
1224 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1225 Illumination correction image.
1227 Returns
1228 -------
1229 result : `lsst.pipe.base.Struct`
1230 Result struct with component:
1231 - ``exposure`` : `afw.image.Exposure`
1232 The fully ISR corrected exposure.
1233 - ``outputExposure`` : `afw.image.Exposure`
1234 An alias for `exposure`
1235 - ``ossThumb`` : `numpy.ndarray`
1236 Thumbnail image of the exposure after overscan subtraction.
1237 - ``flattenedThumb`` : `numpy.ndarray`
1238 Thumbnail image of the exposure after flat-field correction.
1240 Raises
1241 ------
1242 RuntimeError
1243 Raised if a configuration option is set to True, but the
1244 required calibration data has not been specified.
1246 Notes
1247 -----
1248 The current processed exposure can be viewed by setting the
1249 appropriate lsstDebug entries in the `debug.display`
1250 dictionary. The names of these entries correspond to some of
1251 the IsrTaskConfig Boolean options, with the value denoting the
1252 frame to use. The exposure is shown inside the matching
1253 option check and after the processing of that step has
1254 finished. The steps with debug points are:
1256 doAssembleCcd
1257 doBias
1258 doCrosstalk
1259 doBrighterFatter
1260 doDark
1261 doFringe
1262 doStrayLight
1263 doFlat
1265 In addition, setting the "postISRCCD" entry displays the
1266 exposure after all ISR processing has finished.
1268 """
1270 if isGen3 is True:
1271 # Gen3 currently cannot automatically do configuration overrides.
1272 # DM-15257 looks to discuss this issue.
1273 # Configure input exposures;
1274 if detectorNum is None:
1275 raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1277 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1278 bias = self.ensureExposure(bias, camera, detectorNum)
1279 dark = self.ensureExposure(dark, camera, detectorNum)
1280 flat = self.ensureExposure(flat, camera, detectorNum)
1281 else:
1282 if isinstance(ccdExposure, ButlerDataRef):
1283 return self.runDataRef(ccdExposure)
1285 ccd = ccdExposure.getDetector()
1286 filterName = afwImage.Filter(ccdExposure.getFilter().getId()).getName() # Canonical name for filter
1288 if not ccd:
1289 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1290 ccd = [FakeAmp(ccdExposure, self.config)]
1292 # Validate Input
1293 if self.config.doBias and bias is None:
1294 raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1295 if self.doLinearize(ccd) and linearizer is None:
1296 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1297 if self.config.doBrighterFatter and bfKernel is None:
1298 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1299 if self.config.doDark and dark is None:
1300 raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1301 if self.config.doFlat and flat is None:
1302 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1303 if self.config.doDefect and defects is None:
1304 raise RuntimeError("Must supply defects if config.doDefect=True.")
1305 if (self.config.doFringe and filterName in self.fringe.config.filters
1306 and fringes.fringes is None):
1307 # The `fringes` object needs to be a pipeBase.Struct, as
1308 # we use it as a `dict` for the parameters of
1309 # `FringeTask.run()`. The `fringes.fringes` `list` may
1310 # not be `None` if `doFringe=True`. Otherwise, raise.
1311 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1312 if (self.config.doIlluminationCorrection and filterName in self.config.illumFilters
1313 and illumMaskedImage is None):
1314 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1316 # Begin ISR processing.
1317 if self.config.doConvertIntToFloat:
1318 self.log.info("Converting exposure to floating point values.")
1319 ccdExposure = self.convertIntToFloat(ccdExposure)
1321 # Amplifier level processing.
1322 overscans = []
1323 for amp in ccd:
1324 # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1325 if ccdExposure.getBBox().contains(amp.getBBox()):
1326 # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1327 badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1329 if self.config.doOverscan and not badAmp:
1330 # Overscan correction on amp-by-amp basis.
1331 overscanResults = self.overscanCorrection(ccdExposure, amp)
1332 self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1333 if overscanResults is not None and \
1334 self.config.qa is not None and self.config.qa.saveStats is True:
1335 if isinstance(overscanResults.overscanFit, float):
1336 qaMedian = overscanResults.overscanFit
1337 qaStdev = float("NaN")
1338 else:
1339 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1340 afwMath.MEDIAN | afwMath.STDEVCLIP)
1341 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1342 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1344 self.metadata.set(f"ISR OSCAN {amp.getName()} MEDIAN", qaMedian)
1345 self.metadata.set(f"ISR OSCAN {amp.getName()} STDEV", qaStdev)
1346 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1347 amp.getName(), qaMedian, qaStdev)
1348 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1349 else:
1350 if badAmp:
1351 self.log.warn("Amplifier %s is bad.", amp.getName())
1352 overscanResults = None
1354 overscans.append(overscanResults if overscanResults is not None else None)
1355 else:
1356 self.log.info("Skipped OSCAN for %s.", amp.getName())
1358 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1359 self.log.info("Applying crosstalk correction.")
1360 self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1361 crosstalkSources=crosstalkSources)
1362 self.debugView(ccdExposure, "doCrosstalk")
1364 if self.config.doAssembleCcd:
1365 self.log.info("Assembling CCD from amplifiers.")
1366 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1368 if self.config.expectWcs and not ccdExposure.getWcs():
1369 self.log.warn("No WCS found in input exposure.")
1370 self.debugView(ccdExposure, "doAssembleCcd")
1372 ossThumb = None
1373 if self.config.qa.doThumbnailOss:
1374 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1376 if self.config.doBias:
1377 self.log.info("Applying bias correction.")
1378 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1379 trimToFit=self.config.doTrimToMatchCalib)
1380 self.debugView(ccdExposure, "doBias")
1382 if self.config.doVariance:
1383 for amp, overscanResults in zip(ccd, overscans):
1384 if ccdExposure.getBBox().contains(amp.getBBox()):
1385 self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1386 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1387 if overscanResults is not None:
1388 self.updateVariance(ampExposure, amp,
1389 overscanImage=overscanResults.overscanImage)
1390 else:
1391 self.updateVariance(ampExposure, amp,
1392 overscanImage=None)
1393 if self.config.qa is not None and self.config.qa.saveStats is True:
1394 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1395 afwMath.MEDIAN | afwMath.STDEVCLIP)
1396 self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1397 qaStats.getValue(afwMath.MEDIAN))
1398 self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1399 qaStats.getValue(afwMath.STDEVCLIP))
1400 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1401 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1402 qaStats.getValue(afwMath.STDEVCLIP))
1404 if self.doLinearize(ccd):
1405 self.log.info("Applying linearizer.")
1406 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1407 detector=ccd, log=self.log)
1409 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1410 self.log.info("Applying crosstalk correction.")
1411 self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1412 crosstalkSources=crosstalkSources, isTrimmed=True)
1413 self.debugView(ccdExposure, "doCrosstalk")
1415 # Masking block. Optionally mask known defects, NAN pixels, widen trails, and do
1416 # anything else the camera needs. Saturated and suspect pixels have already been masked.
1417 if self.config.doDefect:
1418 self.log.info("Masking defects.")
1419 self.maskDefect(ccdExposure, defects)
1421 if self.config.numEdgeSuspect > 0:
1422 self.log.info("Masking edges as SUSPECT.")
1423 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1424 maskPlane="SUSPECT")
1426 if self.config.doNanMasking:
1427 self.log.info("Masking NAN value pixels.")
1428 self.maskNan(ccdExposure)
1430 if self.config.doWidenSaturationTrails:
1431 self.log.info("Widening saturation trails.")
1432 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1434 if self.config.doCameraSpecificMasking:
1435 self.log.info("Masking regions for camera specific reasons.")
1436 self.masking.run(ccdExposure)
1438 if self.config.doBrighterFatter:
1439 # We need to apply flats and darks before we can interpolate, and we
1440 # need to interpolate before we do B-F, but we do B-F without the
1441 # flats and darks applied so we can work in units of electrons or holes.
1442 # This context manager applies and then removes the darks and flats.
1443 #
1444 # We also do not want to interpolate values here, so operate on temporary
1445 # images so we can apply only the BF-correction and roll back the
1446 # interpolation.
1447 interpExp = ccdExposure.clone()
1448 with self.flatContext(interpExp, flat, dark):
1449 isrFunctions.interpolateFromMask(
1450 maskedImage=interpExp.getMaskedImage(),
1451 fwhm=self.config.fwhm,
1452 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1453 maskNameList=self.config.maskListToInterpolate
1454 )
1455 bfExp = interpExp.clone()
1457 self.log.info("Applying brighter fatter correction using kernel type %s / gains %s.",
1458 type(bfKernel), type(bfGains))
1459 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1460 self.config.brighterFatterMaxIter,
1461 self.config.brighterFatterThreshold,
1462 self.config.brighterFatterApplyGain,
1463 bfGains)
1464 if bfResults[1] == self.config.brighterFatterMaxIter:
1465 self.log.warn("Brighter fatter correction did not converge, final difference %f.",
1466 bfResults[0])
1467 else:
1468 self.log.info("Finished brighter fatter correction in %d iterations.",
1469 bfResults[1])
1470 image = ccdExposure.getMaskedImage().getImage()
1471 bfCorr = bfExp.getMaskedImage().getImage()
1472 bfCorr -= interpExp.getMaskedImage().getImage()
1473 image += bfCorr
1475 # Applying the brighter-fatter correction applies a
1476 # convolution to the science image. At the edges this
1477 # convolution may not have sufficient valid pixels to
1478 # produce a valid correction. Mark pixels within the size
1479 # of the brighter-fatter kernel as EDGE to warn of this
1480 # fact.
1481 self.log.info("Ensuring image edges are masked as SUSPECT to the brighter-fatter kernel size.")
1482 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1483 maskPlane="EDGE")
1485 if self.config.brighterFatterMaskGrowSize > 0:
1486 self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1487 for maskPlane in self.config.maskListToInterpolate:
1488 isrFunctions.growMasks(ccdExposure.getMask(),
1489 radius=self.config.brighterFatterMaskGrowSize,
1490 maskNameList=maskPlane,
1491 maskValue=maskPlane)
1493 self.debugView(ccdExposure, "doBrighterFatter")
1495 if self.config.doDark:
1496 self.log.info("Applying dark correction.")
1497 self.darkCorrection(ccdExposure, dark)
1498 self.debugView(ccdExposure, "doDark")
1500 if self.config.doFringe and not self.config.fringeAfterFlat:
1501 self.log.info("Applying fringe correction before flat.")
1502 self.fringe.run(ccdExposure, **fringes.getDict())
1503 self.debugView(ccdExposure, "doFringe")
1505 if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1506 self.log.info("Checking strayLight correction.")
1507 self.strayLight.run(ccdExposure, strayLightData)
1508 self.debugView(ccdExposure, "doStrayLight")
1510 if self.config.doFlat:
1511 self.log.info("Applying flat correction.")
1512 self.flatCorrection(ccdExposure, flat)
1513 self.debugView(ccdExposure, "doFlat")
1515 if self.config.doApplyGains:
1516 self.log.info("Applying gain correction instead of flat.")
1517 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1519 if self.config.doFringe and self.config.fringeAfterFlat:
1520 self.log.info("Applying fringe correction after flat.")
1521 self.fringe.run(ccdExposure, **fringes.getDict())
1523 if self.config.doVignette:
1524 self.log.info("Constructing Vignette polygon.")
1525 self.vignettePolygon = self.vignette.run(ccdExposure)
1527 if self.config.vignette.doWriteVignettePolygon:
1528 self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1530 if self.config.doAttachTransmissionCurve:
1531 self.log.info("Adding transmission curves.")
1532 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1533 filterTransmission=filterTransmission,
1534 sensorTransmission=sensorTransmission,
1535 atmosphereTransmission=atmosphereTransmission)
1537 flattenedThumb = None
1538 if self.config.qa.doThumbnailFlattened:
1539 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1541 if self.config.doIlluminationCorrection and filterName in self.config.illumFilters:
1542 self.log.info("Performing illumination correction.")
1543 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1544 illumMaskedImage, illumScale=self.config.illumScale,
1545 trimToFit=self.config.doTrimToMatchCalib)
1547 preInterpExp = None
1548 if self.config.doSaveInterpPixels:
1549 preInterpExp = ccdExposure.clone()
1551 # Reset and interpolate bad pixels.
1552 #
1553 # Large contiguous bad regions (which should have the BAD mask
1554 # bit set) should have their values set to the image median.
1555 # This group should include defects and bad amplifiers. As the
1556 # area covered by these defects are large, there's little
1557 # reason to expect that interpolation would provide a more
1558 # useful value.
1559 #
1560 # Smaller defects can be safely interpolated after the larger
1561 # regions have had their pixel values reset. This ensures
1562 # that the remaining defects adjacent to bad amplifiers (as an
1563 # example) do not attempt to interpolate extreme values.
1564 if self.config.doSetBadRegions:
1565 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1566 if badPixelCount > 0:
1567 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1569 if self.config.doInterpolate:
1570 self.log.info("Interpolating masked pixels.")
1571 isrFunctions.interpolateFromMask(
1572 maskedImage=ccdExposure.getMaskedImage(),
1573 fwhm=self.config.fwhm,
1574 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1575 maskNameList=list(self.config.maskListToInterpolate)
1576 )
1578 self.roughZeroPoint(ccdExposure)
1580 if self.config.doMeasureBackground:
1581 self.log.info("Measuring background level.")
1582 self.measureBackground(ccdExposure, self.config.qa)
1584 if self.config.qa is not None and self.config.qa.saveStats is True:
1585 for amp in ccd:
1586 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1587 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1588 afwMath.MEDIAN | afwMath.STDEVCLIP)
1589 self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1590 qaStats.getValue(afwMath.MEDIAN))
1591 self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1592 qaStats.getValue(afwMath.STDEVCLIP))
1593 self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1594 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1595 qaStats.getValue(afwMath.STDEVCLIP))
1597 self.debugView(ccdExposure, "postISRCCD")
1599 return pipeBase.Struct(
1600 exposure=ccdExposure,
1601 ossThumb=ossThumb,
1602 flattenedThumb=flattenedThumb,
1604 preInterpolatedExposure=preInterpExp,
1605 outputExposure=ccdExposure,
1606 outputOssThumbnail=ossThumb,
1607 outputFlattenedThumbnail=flattenedThumb,
1608 )
1610 @pipeBase.timeMethod
1611 def runDataRef(self, sensorRef):
1612 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1614 This method contains the `CmdLineTask` interface to the ISR
1615 processing. All IO is handled here, freeing the `run()` method
1616 to manage only pixel-level calculations. The steps performed
1617 are:
1618 - Read in necessary detrending/isr/calibration data.
1619 - Process raw exposure in `run()`.
1620 - Persist the ISR-corrected exposure as "postISRCCD" if
1621 config.doWrite=True.
1623 Parameters
1624 ----------
1625 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1626 DataRef of the detector data to be processed
1628 Returns
1629 -------
1630 result : `lsst.pipe.base.Struct`
1631 Result struct with component:
1632 - ``exposure`` : `afw.image.Exposure`
1633 The fully ISR corrected exposure.
1635 Raises
1636 ------
1637 RuntimeError
1638 Raised if a configuration option is set to True, but the
1639 required calibration data does not exist.
1641 """
1642 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1644 ccdExposure = sensorRef.get(self.config.datasetType)
1646 camera = sensorRef.get("camera")
1647 isrData = self.readIsrData(sensorRef, ccdExposure)
1649 result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1651 if self.config.doWrite:
1652 sensorRef.put(result.exposure, "postISRCCD")
1653 if result.preInterpolatedExposure is not None:
1654 sensorRef.put(result.preInterpolatedExposure, "postISRCCD_uninterpolated")
1655 if result.ossThumb is not None:
1656 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1657 if result.flattenedThumb is not None:
1658 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1660 return result
1662 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1663 """!Retrieve a calibration dataset for removing instrument signature.
1665 Parameters
1666 ----------
1668 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1669 DataRef of the detector data to find calibration datasets
1670 for.
1671 datasetType : `str`
1672 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1673 dateObs : `str`, optional
1674 Date of the observation. Used to correct butler failures
1675 when using fallback filters.
1676 immediate : `Bool`
1677 If True, disable butler proxies to enable error handling
1678 within this routine.
1680 Returns
1681 -------
1682 exposure : `lsst.afw.image.Exposure`
1683 Requested calibration frame.
1685 Raises
1686 ------
1687 RuntimeError
1688 Raised if no matching calibration frame can be found.
1689 """
1690 try:
1691 exp = dataRef.get(datasetType, immediate=immediate)
1692 except Exception as exc1:
1693 if not self.config.fallbackFilterName:
1694 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1695 try:
1696 if self.config.useFallbackDate and dateObs:
1697 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1698 dateObs=dateObs, immediate=immediate)
1699 else:
1700 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1701 except Exception as exc2:
1702 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1703 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1704 self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1706 if self.config.doAssembleIsrExposures:
1707 exp = self.assembleCcd.assembleCcd(exp)
1708 return exp
1710 def ensureExposure(self, inputExp, camera, detectorNum):
1711 """Ensure that the data returned by Butler is a fully constructed exposure.
1713 ISR requires exposure-level image data for historical reasons, so if we did
1714 not recieve that from Butler, construct it from what we have, modifying the
1715 input in place.
1717 Parameters
1718 ----------
1719 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1720 `lsst.afw.image.ImageF`
1721 The input data structure obtained from Butler.
1722 camera : `lsst.afw.cameraGeom.camera`
1723 The camera associated with the image. Used to find the appropriate
1724 detector.
1725 detectorNum : `int`
1726 The detector this exposure should match.
1728 Returns
1729 -------
1730 inputExp : `lsst.afw.image.Exposure`
1731 The re-constructed exposure, with appropriate detector parameters.
1733 Raises
1734 ------
1735 TypeError
1736 Raised if the input data cannot be used to construct an exposure.
1737 """
1738 if isinstance(inputExp, afwImage.DecoratedImageU):
1739 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1740 elif isinstance(inputExp, afwImage.ImageF):
1741 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1742 elif isinstance(inputExp, afwImage.MaskedImageF):
1743 inputExp = afwImage.makeExposure(inputExp)
1744 elif isinstance(inputExp, afwImage.Exposure):
1745 pass
1746 elif inputExp is None:
1747 # Assume this will be caught by the setup if it is a problem.
1748 return inputExp
1749 else:
1750 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1751 (type(inputExp), ))
1753 if inputExp.getDetector() is None:
1754 inputExp.setDetector(camera[detectorNum])
1756 return inputExp
1758 def convertIntToFloat(self, exposure):
1759 """Convert exposure image from uint16 to float.
1761 If the exposure does not need to be converted, the input is
1762 immediately returned. For exposures that are converted to use
1763 floating point pixels, the variance is set to unity and the
1764 mask to zero.
1766 Parameters
1767 ----------
1768 exposure : `lsst.afw.image.Exposure`
1769 The raw exposure to be converted.
1771 Returns
1772 -------
1773 newexposure : `lsst.afw.image.Exposure`
1774 The input ``exposure``, converted to floating point pixels.
1776 Raises
1777 ------
1778 RuntimeError
1779 Raised if the exposure type cannot be converted to float.
1781 """
1782 if isinstance(exposure, afwImage.ExposureF):
1783 # Nothing to be done
1784 self.log.debug("Exposure already of type float.")
1785 return exposure
1786 if not hasattr(exposure, "convertF"):
1787 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1789 newexposure = exposure.convertF()
1790 newexposure.variance[:] = 1
1791 newexposure.mask[:] = 0x0
1793 return newexposure
1795 def maskAmplifier(self, ccdExposure, amp, defects):
1796 """Identify bad amplifiers, saturated and suspect pixels.
1798 Parameters
1799 ----------
1800 ccdExposure : `lsst.afw.image.Exposure`
1801 Input exposure to be masked.
1802 amp : `lsst.afw.table.AmpInfoCatalog`
1803 Catalog of parameters defining the amplifier on this
1804 exposure to mask.
1805 defects : `lsst.meas.algorithms.Defects`
1806 List of defects. Used to determine if the entire
1807 amplifier is bad.
1809 Returns
1810 -------
1811 badAmp : `Bool`
1812 If this is true, the entire amplifier area is covered by
1813 defects and unusable.
1815 """
1816 maskedImage = ccdExposure.getMaskedImage()
1818 badAmp = False
1820 # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1821 # comparison with current defects definition.
1822 if defects is not None:
1823 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1825 # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1826 # association with pixels in current ccdExposure).
1827 if badAmp:
1828 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1829 afwImage.PARENT)
1830 maskView = dataView.getMask()
1831 maskView |= maskView.getPlaneBitMask("BAD")
1832 del maskView
1833 return badAmp
1835 # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1836 # Saturation and suspect pixels can be masked now, though.
1837 limits = dict()
1838 if self.config.doSaturation and not badAmp:
1839 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1840 if self.config.doSuspect and not badAmp:
1841 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1842 if math.isfinite(self.config.saturation):
1843 limits.update({self.config.saturatedMaskName: self.config.saturation})
1845 for maskName, maskThreshold in limits.items():
1846 if not math.isnan(maskThreshold):
1847 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1848 isrFunctions.makeThresholdMask(
1849 maskedImage=dataView,
1850 threshold=maskThreshold,
1851 growFootprints=0,
1852 maskName=maskName
1853 )
1855 # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1856 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1857 afwImage.PARENT)
1858 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1859 self.config.suspectMaskName])
1860 if numpy.all(maskView.getArray() & maskVal > 0):
1861 badAmp = True
1862 maskView |= maskView.getPlaneBitMask("BAD")
1864 return badAmp
1866 def overscanCorrection(self, ccdExposure, amp):
1867 """Apply overscan correction in place.
1869 This method does initial pixel rejection of the overscan
1870 region. The overscan can also be optionally segmented to
1871 allow for discontinuous overscan responses to be fit
1872 separately. The actual overscan subtraction is performed by
1873 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1874 which is called here after the amplifier is preprocessed.
1876 Parameters
1877 ----------
1878 ccdExposure : `lsst.afw.image.Exposure`
1879 Exposure to have overscan correction performed.
1880 amp : `lsst.afw.table.AmpInfoCatalog`
1881 The amplifier to consider while correcting the overscan.
1883 Returns
1884 -------
1885 overscanResults : `lsst.pipe.base.Struct`
1886 Result struct with components:
1887 - ``imageFit`` : scalar or `lsst.afw.image.Image`
1888 Value or fit subtracted from the amplifier image data.
1889 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
1890 Value or fit subtracted from the overscan image data.
1891 - ``overscanImage`` : `lsst.afw.image.Image`
1892 Image of the overscan region with the overscan
1893 correction applied. This quantity is used to estimate
1894 the amplifier read noise empirically.
1896 Raises
1897 ------
1898 RuntimeError
1899 Raised if the ``amp`` does not contain raw pixel information.
1901 See Also
1902 --------
1903 lsst.ip.isr.isrFunctions.overscanCorrection
1904 """
1905 if amp.getRawHorizontalOverscanBBox().isEmpty():
1906 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
1907 return None
1909 statControl = afwMath.StatisticsControl()
1910 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1912 # Determine the bounding boxes
1913 dataBBox = amp.getRawDataBBox()
1914 oscanBBox = amp.getRawHorizontalOverscanBBox()
1915 dx0 = 0
1916 dx1 = 0
1918 prescanBBox = amp.getRawPrescanBBox()
1919 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
1920 dx0 += self.config.overscanNumLeadingColumnsToSkip
1921 dx1 -= self.config.overscanNumTrailingColumnsToSkip
1922 else:
1923 dx0 += self.config.overscanNumTrailingColumnsToSkip
1924 dx1 -= self.config.overscanNumLeadingColumnsToSkip
1926 # Determine if we need to work on subregions of the amplifier and overscan.
1927 imageBBoxes = []
1928 overscanBBoxes = []
1930 if ((self.config.overscanBiasJump
1931 and self.config.overscanBiasJumpLocation)
1932 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
1933 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
1934 self.config.overscanBiasJumpDevices)):
1935 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
1936 yLower = self.config.overscanBiasJumpLocation
1937 yUpper = dataBBox.getHeight() - yLower
1938 else:
1939 yUpper = self.config.overscanBiasJumpLocation
1940 yLower = dataBBox.getHeight() - yUpper
1942 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1943 lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
1944 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1945 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1946 yLower)))
1948 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
1949 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
1950 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
1951 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1952 yUpper)))
1953 else:
1954 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
1955 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
1956 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
1957 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
1958 oscanBBox.getHeight())))
1960 # Perform overscan correction on subregions, ensuring saturated pixels are masked.
1961 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
1962 ampImage = ccdExposure.maskedImage[imageBBox]
1963 overscanImage = ccdExposure.maskedImage[overscanBBox]
1965 overscanArray = overscanImage.image.array
1966 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
1967 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
1968 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
1970 statControl = afwMath.StatisticsControl()
1971 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
1973 overscanResults = self.overscan.run(ampImage.getImage(), overscanImage)
1975 # Measure average overscan levels and record them in the metadata.
1976 levelStat = afwMath.MEDIAN
1977 sigmaStat = afwMath.STDEVCLIP
1979 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
1980 self.config.qa.flatness.nIter)
1981 metadata = ccdExposure.getMetadata()
1982 ampNum = amp.getName()
1983 # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
1984 if isinstance(overscanResults.overscanFit, float):
1985 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
1986 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
1987 else:
1988 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
1989 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
1990 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
1992 return overscanResults
1994 def updateVariance(self, ampExposure, amp, overscanImage=None):
1995 """Set the variance plane using the amplifier gain and read noise
1997 The read noise is calculated from the ``overscanImage`` if the
1998 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
1999 the value from the amplifier data is used.
2001 Parameters
2002 ----------
2003 ampExposure : `lsst.afw.image.Exposure`
2004 Exposure to process.
2005 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2006 Amplifier detector data.
2007 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2008 Image of overscan, required only for empirical read noise.
2010 See also
2011 --------
2012 lsst.ip.isr.isrFunctions.updateVariance
2013 """
2014 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2015 gain = amp.getGain()
2017 if math.isnan(gain):
2018 gain = 1.0
2019 self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2020 elif gain <= 0:
2021 patchedGain = 1.0
2022 self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
2023 amp.getName(), gain, patchedGain)
2024 gain = patchedGain
2026 if self.config.doEmpiricalReadNoise and overscanImage is None:
2027 self.log.info("Overscan is none for EmpiricalReadNoise.")
2029 if self.config.doEmpiricalReadNoise and overscanImage is not None:
2030 stats = afwMath.StatisticsControl()
2031 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2032 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2033 self.log.info("Calculated empirical read noise for amp %s: %f.",
2034 amp.getName(), readNoise)
2035 else:
2036 readNoise = amp.getReadNoise()
2038 isrFunctions.updateVariance(
2039 maskedImage=ampExposure.getMaskedImage(),
2040 gain=gain,
2041 readNoise=readNoise,
2042 )
2044 def darkCorrection(self, exposure, darkExposure, invert=False):
2045 """!Apply dark correction in place.
2047 Parameters
2048 ----------
2049 exposure : `lsst.afw.image.Exposure`
2050 Exposure to process.
2051 darkExposure : `lsst.afw.image.Exposure`
2052 Dark exposure of the same size as ``exposure``.
2053 invert : `Bool`, optional
2054 If True, re-add the dark to an already corrected image.
2056 Raises
2057 ------
2058 RuntimeError
2059 Raised if either ``exposure`` or ``darkExposure`` do not
2060 have their dark time defined.
2062 See Also
2063 --------
2064 lsst.ip.isr.isrFunctions.darkCorrection
2065 """
2066 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2067 if math.isnan(expScale):
2068 raise RuntimeError("Exposure darktime is NAN.")
2069 if darkExposure.getInfo().getVisitInfo() is not None \
2070 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2071 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2072 else:
2073 # DM-17444: darkExposure.getInfo.getVisitInfo() is None
2074 # so getDarkTime() does not exist.
2075 self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2076 darkScale = 1.0
2078 isrFunctions.darkCorrection(
2079 maskedImage=exposure.getMaskedImage(),
2080 darkMaskedImage=darkExposure.getMaskedImage(),
2081 expScale=expScale,
2082 darkScale=darkScale,
2083 invert=invert,
2084 trimToFit=self.config.doTrimToMatchCalib
2085 )
2087 def doLinearize(self, detector):
2088 """!Check if linearization is needed for the detector cameraGeom.
2090 Checks config.doLinearize and the linearity type of the first
2091 amplifier.
2093 Parameters
2094 ----------
2095 detector : `lsst.afw.cameraGeom.Detector`
2096 Detector to get linearity type from.
2098 Returns
2099 -------
2100 doLinearize : `Bool`
2101 If True, linearization should be performed.
2102 """
2103 return self.config.doLinearize and \
2104 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2106 def flatCorrection(self, exposure, flatExposure, invert=False):
2107 """!Apply flat correction in place.
2109 Parameters
2110 ----------
2111 exposure : `lsst.afw.image.Exposure`
2112 Exposure to process.
2113 flatExposure : `lsst.afw.image.Exposure`
2114 Flat exposure of the same size as ``exposure``.
2115 invert : `Bool`, optional
2116 If True, unflatten an already flattened image.
2118 See Also
2119 --------
2120 lsst.ip.isr.isrFunctions.flatCorrection
2121 """
2122 isrFunctions.flatCorrection(
2123 maskedImage=exposure.getMaskedImage(),
2124 flatMaskedImage=flatExposure.getMaskedImage(),
2125 scalingType=self.config.flatScalingType,
2126 userScale=self.config.flatUserScale,
2127 invert=invert,
2128 trimToFit=self.config.doTrimToMatchCalib
2129 )
2131 def saturationDetection(self, exposure, amp):
2132 """!Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2134 Parameters
2135 ----------
2136 exposure : `lsst.afw.image.Exposure`
2137 Exposure to process. Only the amplifier DataSec is processed.
2138 amp : `lsst.afw.table.AmpInfoCatalog`
2139 Amplifier detector data.
2141 See Also
2142 --------
2143 lsst.ip.isr.isrFunctions.makeThresholdMask
2144 """
2145 if not math.isnan(amp.getSaturation()):
2146 maskedImage = exposure.getMaskedImage()
2147 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2148 isrFunctions.makeThresholdMask(
2149 maskedImage=dataView,
2150 threshold=amp.getSaturation(),
2151 growFootprints=0,
2152 maskName=self.config.saturatedMaskName,
2153 )
2155 def saturationInterpolation(self, exposure):
2156 """!Interpolate over saturated pixels, in place.
2158 This method should be called after `saturationDetection`, to
2159 ensure that the saturated pixels have been identified in the
2160 SAT mask. It should also be called after `assembleCcd`, since
2161 saturated regions may cross amplifier boundaries.
2163 Parameters
2164 ----------
2165 exposure : `lsst.afw.image.Exposure`
2166 Exposure to process.
2168 See Also
2169 --------
2170 lsst.ip.isr.isrTask.saturationDetection
2171 lsst.ip.isr.isrFunctions.interpolateFromMask
2172 """
2173 isrFunctions.interpolateFromMask(
2174 maskedImage=exposure.getMaskedImage(),
2175 fwhm=self.config.fwhm,
2176 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2177 maskNameList=list(self.config.saturatedMaskName),
2178 )
2180 def suspectDetection(self, exposure, amp):
2181 """!Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2183 Parameters
2184 ----------
2185 exposure : `lsst.afw.image.Exposure`
2186 Exposure to process. Only the amplifier DataSec is processed.
2187 amp : `lsst.afw.table.AmpInfoCatalog`
2188 Amplifier detector data.
2190 See Also
2191 --------
2192 lsst.ip.isr.isrFunctions.makeThresholdMask
2194 Notes
2195 -----
2196 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2197 This is intended to indicate pixels that may be affected by unknown systematics;
2198 for example if non-linearity corrections above a certain level are unstable
2199 then that would be a useful value for suspectLevel. A value of `nan` indicates
2200 that no such level exists and no pixels are to be masked as suspicious.
2201 """
2202 suspectLevel = amp.getSuspectLevel()
2203 if math.isnan(suspectLevel):
2204 return
2206 maskedImage = exposure.getMaskedImage()
2207 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2208 isrFunctions.makeThresholdMask(
2209 maskedImage=dataView,
2210 threshold=suspectLevel,
2211 growFootprints=0,
2212 maskName=self.config.suspectMaskName,
2213 )
2215 def maskDefect(self, exposure, defectBaseList):
2216 """!Mask defects using mask plane "BAD", in place.
2218 Parameters
2219 ----------
2220 exposure : `lsst.afw.image.Exposure`
2221 Exposure to process.
2222 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2223 `lsst.afw.image.DefectBase`.
2224 List of defects to mask.
2226 Notes
2227 -----
2228 Call this after CCD assembly, since defects may cross amplifier boundaries.
2229 """
2230 maskedImage = exposure.getMaskedImage()
2231 if not isinstance(defectBaseList, Defects):
2232 # Promotes DefectBase to Defect
2233 defectList = Defects(defectBaseList)
2234 else:
2235 defectList = defectBaseList
2236 defectList.maskPixels(maskedImage, maskName="BAD")
2238 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT"):
2239 """!Mask edge pixels with applicable mask plane.
2241 Parameters
2242 ----------
2243 exposure : `lsst.afw.image.Exposure`
2244 Exposure to process.
2245 numEdgePixels : `int`, optional
2246 Number of edge pixels to mask.
2247 maskPlane : `str`, optional
2248 Mask plane name to use.
2249 """
2250 maskedImage = exposure.getMaskedImage()
2251 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2253 if numEdgePixels > 0:
2254 goodBBox = maskedImage.getBBox()
2255 # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2256 goodBBox.grow(-numEdgePixels)
2257 # Mask pixels outside goodBBox
2258 SourceDetectionTask.setEdgeBits(
2259 maskedImage,
2260 goodBBox,
2261 maskBitMask
2262 )
2264 def maskAndInterpolateDefects(self, exposure, defectBaseList):
2265 """Mask and interpolate defects using mask plane "BAD", in place.
2267 Parameters
2268 ----------
2269 exposure : `lsst.afw.image.Exposure`
2270 Exposure to process.
2271 defectBaseList : `lsst.meas.algorithms.Defects` or `list` of
2272 `lsst.afw.image.DefectBase`.
2273 List of defects to mask and interpolate.
2275 See Also
2276 --------
2277 lsst.ip.isr.isrTask.maskDefect()
2278 """
2279 self.maskDefect(exposure, defectBaseList)
2280 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2281 maskPlane="SUSPECT")
2282 isrFunctions.interpolateFromMask(
2283 maskedImage=exposure.getMaskedImage(),
2284 fwhm=self.config.fwhm,
2285 growSaturatedFootprints=0,
2286 maskNameList=["BAD"],
2287 )
2289 def maskNan(self, exposure):
2290 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2292 Parameters
2293 ----------
2294 exposure : `lsst.afw.image.Exposure`
2295 Exposure to process.
2297 Notes
2298 -----
2299 We mask over all NaNs, including those that are masked with
2300 other bits (because those may or may not be interpolated over
2301 later, and we want to remove all NaNs). Despite this
2302 behaviour, the "UNMASKEDNAN" mask plane is used to preserve
2303 the historical name.
2304 """
2305 maskedImage = exposure.getMaskedImage()
2307 # Find and mask NaNs
2308 maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2309 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2310 numNans = maskNans(maskedImage, maskVal)
2311 self.metadata.set("NUMNANS", numNans)
2312 if numNans > 0:
2313 self.log.warn("There were %d unmasked NaNs.", numNans)
2315 def maskAndInterpolateNan(self, exposure):
2316 """"Mask and interpolate NaNs using mask plane "UNMASKEDNAN", in place.
2318 Parameters
2319 ----------
2320 exposure : `lsst.afw.image.Exposure`
2321 Exposure to process.
2323 See Also
2324 --------
2325 lsst.ip.isr.isrTask.maskNan()
2326 """
2327 self.maskNan(exposure)
2328 isrFunctions.interpolateFromMask(
2329 maskedImage=exposure.getMaskedImage(),
2330 fwhm=self.config.fwhm,
2331 growSaturatedFootprints=0,
2332 maskNameList=["UNMASKEDNAN"],
2333 )
2335 def measureBackground(self, exposure, IsrQaConfig=None):
2336 """Measure the image background in subgrids, for quality control purposes.
2338 Parameters
2339 ----------
2340 exposure : `lsst.afw.image.Exposure`
2341 Exposure to process.
2342 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2343 Configuration object containing parameters on which background
2344 statistics and subgrids to use.
2345 """
2346 if IsrQaConfig is not None:
2347 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2348 IsrQaConfig.flatness.nIter)
2349 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2350 statsControl.setAndMask(maskVal)
2351 maskedImage = exposure.getMaskedImage()
2352 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2353 skyLevel = stats.getValue(afwMath.MEDIAN)
2354 skySigma = stats.getValue(afwMath.STDEVCLIP)
2355 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2356 metadata = exposure.getMetadata()
2357 metadata.set('SKYLEVEL', skyLevel)
2358 metadata.set('SKYSIGMA', skySigma)
2360 # calcluating flatlevel over the subgrids
2361 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2362 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2363 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2364 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2365 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2366 skyLevels = numpy.zeros((nX, nY))
2368 for j in range(nY):
2369 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2370 for i in range(nX):
2371 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2373 xLLC = xc - meshXHalf
2374 yLLC = yc - meshYHalf
2375 xURC = xc + meshXHalf - 1
2376 yURC = yc + meshYHalf - 1
2378 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2379 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2381 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2383 good = numpy.where(numpy.isfinite(skyLevels))
2384 skyMedian = numpy.median(skyLevels[good])
2385 flatness = (skyLevels[good] - skyMedian) / skyMedian
2386 flatness_rms = numpy.std(flatness)
2387 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2389 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2390 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2391 nX, nY, flatness_pp, flatness_rms)
2393 metadata.set('FLATNESS_PP', float(flatness_pp))
2394 metadata.set('FLATNESS_RMS', float(flatness_rms))
2395 metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2396 metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2397 metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2399 def roughZeroPoint(self, exposure):
2400 """Set an approximate magnitude zero point for the exposure.
2402 Parameters
2403 ----------
2404 exposure : `lsst.afw.image.Exposure`
2405 Exposure to process.
2406 """
2407 filterName = afwImage.Filter(exposure.getFilter().getId()).getName() # Canonical name for filter
2408 if filterName in self.config.fluxMag0T1:
2409 fluxMag0 = self.config.fluxMag0T1[filterName]
2410 else:
2411 self.log.warn("No rough magnitude zero point set for filter %s.", filterName)
2412 fluxMag0 = self.config.defaultFluxMag0T1
2414 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2415 if not expTime > 0: # handle NaN as well as <= 0
2416 self.log.warn("Non-positive exposure time; skipping rough zero point.")
2417 return
2419 self.log.info("Setting rough magnitude zero point: %f", 2.5*math.log10(fluxMag0*expTime))
2420 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2422 def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2423 """!Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2425 Parameters
2426 ----------
2427 ccdExposure : `lsst.afw.image.Exposure`
2428 Exposure to process.
2429 fpPolygon : `lsst.afw.geom.Polygon`
2430 Polygon in focal plane coordinates.
2431 """
2432 # Get ccd corners in focal plane coordinates
2433 ccd = ccdExposure.getDetector()
2434 fpCorners = ccd.getCorners(FOCAL_PLANE)
2435 ccdPolygon = Polygon(fpCorners)
2437 # Get intersection of ccd corners with fpPolygon
2438 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2440 # Transform back to pixel positions and build new polygon
2441 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2442 validPolygon = Polygon(ccdPoints)
2443 ccdExposure.getInfo().setValidPolygon(validPolygon)
2445 @contextmanager
2446 def flatContext(self, exp, flat, dark=None):
2447 """Context manager that applies and removes flats and darks,
2448 if the task is configured to apply them.
2450 Parameters
2451 ----------
2452 exp : `lsst.afw.image.Exposure`
2453 Exposure to process.
2454 flat : `lsst.afw.image.Exposure`
2455 Flat exposure the same size as ``exp``.
2456 dark : `lsst.afw.image.Exposure`, optional
2457 Dark exposure the same size as ``exp``.
2459 Yields
2460 ------
2461 exp : `lsst.afw.image.Exposure`
2462 The flat and dark corrected exposure.
2463 """
2464 if self.config.doDark and dark is not None:
2465 self.darkCorrection(exp, dark)
2466 if self.config.doFlat:
2467 self.flatCorrection(exp, flat)
2468 try:
2469 yield exp
2470 finally:
2471 if self.config.doFlat:
2472 self.flatCorrection(exp, flat, invert=True)
2473 if self.config.doDark and dark is not None:
2474 self.darkCorrection(exp, dark, invert=True)
2476 def debugView(self, exposure, stepname):
2477 """Utility function to examine ISR exposure at different stages.
2479 Parameters
2480 ----------
2481 exposure : `lsst.afw.image.Exposure`
2482 Exposure to view.
2483 stepname : `str`
2484 State of processing to view.
2485 """
2486 frame = getDebugFrame(self._display, stepname)
2487 if frame:
2488 display = getDisplay(frame)
2489 display.scale('asinh', 'zscale')
2490 display.mtv(exposure)
2491 prompt = "Press Enter to continue [c]... "
2492 while True:
2493 ans = input(prompt).lower()
2494 if ans in ("", "c",):
2495 break
2498class FakeAmp(object):
2499 """A Detector-like object that supports returning gain and saturation level
2501 This is used when the input exposure does not have a detector.
2503 Parameters
2504 ----------
2505 exposure : `lsst.afw.image.Exposure`
2506 Exposure to generate a fake amplifier for.
2507 config : `lsst.ip.isr.isrTaskConfig`
2508 Configuration to apply to the fake amplifier.
2509 """
2511 def __init__(self, exposure, config):
2512 self._bbox = exposure.getBBox(afwImage.LOCAL)
2513 self._RawHorizontalOverscanBBox = lsst.geom.Box2I()
2514 self._gain = config.gain
2515 self._readNoise = config.readNoise
2516 self._saturation = config.saturation
2518 def getBBox(self):
2519 return self._bbox
2521 def getRawBBox(self):
2522 return self._bbox
2524 def getRawHorizontalOverscanBBox(self):
2525 return self._RawHorizontalOverscanBBox
2527 def getGain(self):
2528 return self._gain
2530 def getReadNoise(self):
2531 return self._readNoise
2533 def getSaturation(self):
2534 return self._saturation
2536 def getSuspectLevel(self):
2537 return float("NaN")
2540class RunIsrConfig(pexConfig.Config):
2541 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2544class RunIsrTask(pipeBase.CmdLineTask):
2545 """Task to wrap the default IsrTask to allow it to be retargeted.
2547 The standard IsrTask can be called directly from a command line
2548 program, but doing so removes the ability of the task to be
2549 retargeted. As most cameras override some set of the IsrTask
2550 methods, this would remove those data-specific methods in the
2551 output post-ISR images. This wrapping class fixes the issue,
2552 allowing identical post-ISR images to be generated by both the
2553 processCcd and isrTask code.
2554 """
2555 ConfigClass = RunIsrConfig
2556 _DefaultName = "runIsr"
2558 def __init__(self, *args, **kwargs):
2559 super().__init__(*args, **kwargs)
2560 self.makeSubtask("isr")
2562 def runDataRef(self, dataRef):
2563 """
2564 Parameters
2565 ----------
2566 dataRef : `lsst.daf.persistence.ButlerDataRef`
2567 data reference of the detector data to be processed
2569 Returns
2570 -------
2571 result : `pipeBase.Struct`
2572 Result struct with component:
2574 - exposure : `lsst.afw.image.Exposure`
2575 Post-ISR processed exposure.
2576 """
2577 return self.isr.runDataRef(dataRef)