Coverage for python/lsst/ip/isr/isrTask.py : 16%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import math
23import numpy
25import lsst.geom
26import lsst.afw.image as afwImage
27import lsst.afw.math as afwMath
28import lsst.pex.config as pexConfig
29import lsst.pipe.base as pipeBase
30import lsst.pipe.base.connectionTypes as cT
32from contextlib import contextmanager
33from lsstDebug import getDebugFrame
35from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36 ReadoutCorner)
37from lsst.afw.display import getDisplay
38from lsst.afw.geom import Polygon
39from lsst.daf.persistence import ButlerDataRef
40from lsst.daf.persistence.butler import NoResults
41from lsst.meas.algorithms.detection import SourceDetectionTask
43from . import isrFunctions
44from . import isrQa
45from . import linearize
46from .defects import Defects
48from .assembleCcdTask import AssembleCcdTask
49from .crosstalk import CrosstalkTask, CrosstalkCalib
50from .fringe import FringeTask
51from .isr import maskNans
52from .masking import MaskingTask
53from .overscan import OverscanCorrectionTask
54from .straylight import StrayLightTask
55from .vignette import VignetteTask
56from lsst.daf.butler import DimensionGraph
59__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
62def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections):
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
67 populated.
69 This will be unused until DM-25348 resolves the quantum graph
70 generation issue.
72 Parameters
73 ----------
74 datasetType : `str`
75 Dataset to lookup.
76 registry : `lsst.daf.butler.Registry`
77 Butler registry to query.
78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79 Data id to transform to identify crosstalkSources. The
80 ``detector`` entry will be stripped.
81 collections : `lsst.daf.butler.CollectionSearch`
82 Collections to search through.
84 Returns
85 -------
86 results : `list` [`lsst.daf.butler.DatasetRef`]
87 List of datasets that match the query that will be used as
88 crosstalkSources.
89 """
90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"]))
91 results = list(registry.queryDatasets(datasetType,
92 collections=collections,
93 dataId=newDataId,
94 findFirst=True,
95 ).expanded())
96 return results
99class IsrTaskConnections(pipeBase.PipelineTaskConnections,
100 dimensions={"instrument", "exposure", "detector"},
101 defaultTemplates={}):
102 ccdExposure = cT.Input(
103 name="raw",
104 doc="Input exposure to process.",
105 storageClass="Exposure",
106 dimensions=["instrument", "exposure", "detector"],
107 )
108 camera = cT.PrerequisiteInput(
109 name="camera",
110 storageClass="Camera",
111 doc="Input camera to construct complete exposures.",
112 dimensions=["instrument"],
113 isCalibration=True,
114 )
116 crosstalk = cT.PrerequisiteInput(
117 name="crosstalk",
118 doc="Input crosstalk object",
119 storageClass="CrosstalkCalib",
120 dimensions=["instrument", "detector"],
121 isCalibration=True,
122 minimum=0, # can fall back to cameraGeom
123 )
124 # TODO: DM-25348. This does not work yet to correctly load
125 # possible crosstalk sources.
126 crosstalkSources = cT.PrerequisiteInput(
127 name="isrOverscanCorrected",
128 doc="Overscan corrected input images.",
129 storageClass="Exposure",
130 dimensions=["instrument", "exposure", "detector"],
131 deferLoad=True,
132 multiple=True,
133 lookupFunction=crosstalkSourceLookup,
134 minimum=0, # not needed for all instruments, no config to control this
135 )
136 bias = cT.PrerequisiteInput(
137 name="bias",
138 doc="Input bias calibration.",
139 storageClass="ExposureF",
140 dimensions=["instrument", "detector"],
141 isCalibration=True,
142 )
143 dark = cT.PrerequisiteInput(
144 name='dark',
145 doc="Input dark calibration.",
146 storageClass="ExposureF",
147 dimensions=["instrument", "detector"],
148 isCalibration=True,
149 )
150 flat = cT.PrerequisiteInput(
151 name="flat",
152 doc="Input flat calibration.",
153 storageClass="ExposureF",
154 dimensions=["instrument", "physical_filter", "detector"],
155 isCalibration=True,
156 )
157 ptc = cT.PrerequisiteInput(
158 name="ptc",
159 doc="Input Photon Transfer Curve dataset",
160 storageClass="PhotonTransferCurveDataset",
161 dimensions=["instrument", "detector"],
162 isCalibration=True,
163 )
164 fringes = cT.PrerequisiteInput(
165 name="fringe",
166 doc="Input fringe calibration.",
167 storageClass="ExposureF",
168 dimensions=["instrument", "physical_filter", "detector"],
169 isCalibration=True,
170 minimum=0, # only needed for some bands, even when enabled
171 )
172 strayLightData = cT.PrerequisiteInput(
173 name='yBackground',
174 doc="Input stray light calibration.",
175 storageClass="StrayLightData",
176 dimensions=["instrument", "physical_filter", "detector"],
177 isCalibration=True,
178 minimum=0, # only needed for some bands, even when enabled
179 )
180 bfKernel = cT.PrerequisiteInput(
181 name='bfKernel',
182 doc="Input brighter-fatter kernel.",
183 storageClass="NumpyArray",
184 dimensions=["instrument"],
185 isCalibration=True,
186 minimum=0, # can use either bfKernel or newBFKernel
187 )
188 newBFKernel = cT.PrerequisiteInput(
189 name='brighterFatterKernel',
190 doc="Newer complete kernel + gain solutions.",
191 storageClass="BrighterFatterKernel",
192 dimensions=["instrument", "detector"],
193 isCalibration=True,
194 minimum=0, # can use either bfKernel or newBFKernel
195 )
196 defects = cT.PrerequisiteInput(
197 name='defects',
198 doc="Input defect tables.",
199 storageClass="Defects",
200 dimensions=["instrument", "detector"],
201 isCalibration=True,
202 )
203 linearizer = cT.PrerequisiteInput(
204 name='linearizer',
205 storageClass="Linearizer",
206 doc="Linearity correction calibration.",
207 dimensions=["instrument", "detector"],
208 isCalibration=True,
209 minimum=0, # can fall back to cameraGeom
210 )
211 opticsTransmission = cT.PrerequisiteInput(
212 name="transmission_optics",
213 storageClass="TransmissionCurve",
214 doc="Transmission curve due to the optics.",
215 dimensions=["instrument"],
216 isCalibration=True,
217 )
218 filterTransmission = cT.PrerequisiteInput(
219 name="transmission_filter",
220 storageClass="TransmissionCurve",
221 doc="Transmission curve due to the filter.",
222 dimensions=["instrument", "physical_filter"],
223 isCalibration=True,
224 )
225 sensorTransmission = cT.PrerequisiteInput(
226 name="transmission_sensor",
227 storageClass="TransmissionCurve",
228 doc="Transmission curve due to the sensor.",
229 dimensions=["instrument", "detector"],
230 isCalibration=True,
231 )
232 atmosphereTransmission = cT.PrerequisiteInput(
233 name="transmission_atmosphere",
234 storageClass="TransmissionCurve",
235 doc="Transmission curve due to the atmosphere.",
236 dimensions=["instrument"],
237 isCalibration=True,
238 )
239 illumMaskedImage = cT.PrerequisiteInput(
240 name="illum",
241 doc="Input illumination correction.",
242 storageClass="MaskedImageF",
243 dimensions=["instrument", "physical_filter", "detector"],
244 isCalibration=True,
245 )
247 outputExposure = cT.Output(
248 name='postISRCCD',
249 doc="Output ISR processed exposure.",
250 storageClass="Exposure",
251 dimensions=["instrument", "exposure", "detector"],
252 )
253 preInterpExposure = cT.Output(
254 name='preInterpISRCCD',
255 doc="Output ISR processed exposure, with pixels left uninterpolated.",
256 storageClass="ExposureF",
257 dimensions=["instrument", "exposure", "detector"],
258 )
259 outputOssThumbnail = cT.Output(
260 name="OssThumb",
261 doc="Output Overscan-subtracted thumbnail image.",
262 storageClass="Thumbnail",
263 dimensions=["instrument", "exposure", "detector"],
264 )
265 outputFlattenedThumbnail = cT.Output(
266 name="FlattenedThumb",
267 doc="Output flat-corrected thumbnail image.",
268 storageClass="Thumbnail",
269 dimensions=["instrument", "exposure", "detector"],
270 )
272 def __init__(self, *, config=None):
273 super().__init__(config=config)
275 if config.doBias is not True:
276 self.prerequisiteInputs.discard("bias")
277 if config.doLinearize is not True:
278 self.prerequisiteInputs.discard("linearizer")
279 if config.doCrosstalk is not True:
280 self.inputs.discard("crosstalkSources")
281 self.prerequisiteInputs.discard("crosstalk")
282 if config.doBrighterFatter is not True:
283 self.prerequisiteInputs.discard("bfKernel")
284 self.prerequisiteInputs.discard("newBFKernel")
285 if config.doDefect is not True:
286 self.prerequisiteInputs.discard("defects")
287 if config.doDark is not True:
288 self.prerequisiteInputs.discard("dark")
289 if config.doFlat is not True:
290 self.prerequisiteInputs.discard("flat")
291 if config.doFringe is not True:
292 self.prerequisiteInputs.discard("fringe")
293 if config.doStrayLight is not True:
294 self.prerequisiteInputs.discard("strayLightData")
295 if config.usePtcGains is not True and config.usePtcReadNoise is not True:
296 self.prerequisiteInputs.discard("ptc")
297 if config.doAttachTransmissionCurve is not True:
298 self.prerequisiteInputs.discard("opticsTransmission")
299 self.prerequisiteInputs.discard("filterTransmission")
300 self.prerequisiteInputs.discard("sensorTransmission")
301 self.prerequisiteInputs.discard("atmosphereTransmission")
302 if config.doUseOpticsTransmission is not True:
303 self.prerequisiteInputs.discard("opticsTransmission")
304 if config.doUseFilterTransmission is not True:
305 self.prerequisiteInputs.discard("filterTransmission")
306 if config.doUseSensorTransmission is not True:
307 self.prerequisiteInputs.discard("sensorTransmission")
308 if config.doUseAtmosphereTransmission is not True:
309 self.prerequisiteInputs.discard("atmosphereTransmission")
310 if config.doIlluminationCorrection is not True:
311 self.prerequisiteInputs.discard("illumMaskedImage")
313 if config.doWrite is not True:
314 self.outputs.discard("outputExposure")
315 self.outputs.discard("preInterpExposure")
316 self.outputs.discard("outputFlattenedThumbnail")
317 self.outputs.discard("outputOssThumbnail")
318 if config.doSaveInterpPixels is not True:
319 self.outputs.discard("preInterpExposure")
320 if config.qa.doThumbnailOss is not True:
321 self.outputs.discard("outputOssThumbnail")
322 if config.qa.doThumbnailFlattened is not True:
323 self.outputs.discard("outputFlattenedThumbnail")
326class IsrTaskConfig(pipeBase.PipelineTaskConfig,
327 pipelineConnections=IsrTaskConnections):
328 """Configuration parameters for IsrTask.
330 Items are grouped in the order in which they are executed by the task.
331 """
332 datasetType = pexConfig.Field(
333 dtype=str,
334 doc="Dataset type for input data; users will typically leave this alone, "
335 "but camera-specific ISR tasks will override it",
336 default="raw",
337 )
339 fallbackFilterName = pexConfig.Field(
340 dtype=str,
341 doc="Fallback default filter name for calibrations.",
342 optional=True
343 )
344 useFallbackDate = pexConfig.Field(
345 dtype=bool,
346 doc="Pass observation date when using fallback filter.",
347 default=False,
348 )
349 expectWcs = pexConfig.Field(
350 dtype=bool,
351 default=True,
352 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
353 )
354 fwhm = pexConfig.Field(
355 dtype=float,
356 doc="FWHM of PSF in arcseconds.",
357 default=1.0,
358 )
359 qa = pexConfig.ConfigField(
360 dtype=isrQa.IsrQaConfig,
361 doc="QA related configuration options.",
362 )
364 # Image conversion configuration
365 doConvertIntToFloat = pexConfig.Field(
366 dtype=bool,
367 doc="Convert integer raw images to floating point values?",
368 default=True,
369 )
371 # Saturated pixel handling.
372 doSaturation = pexConfig.Field(
373 dtype=bool,
374 doc="Mask saturated pixels? NB: this is totally independent of the"
375 " interpolation option - this is ONLY setting the bits in the mask."
376 " To have them interpolated make sure doSaturationInterpolation=True",
377 default=True,
378 )
379 saturatedMaskName = pexConfig.Field(
380 dtype=str,
381 doc="Name of mask plane to use in saturation detection and interpolation",
382 default="SAT",
383 )
384 saturation = pexConfig.Field(
385 dtype=float,
386 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
387 default=float("NaN"),
388 )
389 growSaturationFootprintSize = pexConfig.Field(
390 dtype=int,
391 doc="Number of pixels by which to grow the saturation footprints",
392 default=1,
393 )
395 # Suspect pixel handling.
396 doSuspect = pexConfig.Field(
397 dtype=bool,
398 doc="Mask suspect pixels?",
399 default=False,
400 )
401 suspectMaskName = pexConfig.Field(
402 dtype=str,
403 doc="Name of mask plane to use for suspect pixels",
404 default="SUSPECT",
405 )
406 numEdgeSuspect = pexConfig.Field(
407 dtype=int,
408 doc="Number of edge pixels to be flagged as untrustworthy.",
409 default=0,
410 )
411 edgeMaskLevel = pexConfig.ChoiceField(
412 dtype=str,
413 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
414 default="DETECTOR",
415 allowed={
416 'DETECTOR': 'Mask only the edges of the full detector.',
417 'AMP': 'Mask edges of each amplifier.',
418 },
419 )
421 # Initial masking options.
422 doSetBadRegions = pexConfig.Field(
423 dtype=bool,
424 doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
425 default=True,
426 )
427 badStatistic = pexConfig.ChoiceField(
428 dtype=str,
429 doc="How to estimate the average value for BAD regions.",
430 default='MEANCLIP',
431 allowed={
432 "MEANCLIP": "Correct using the (clipped) mean of good data",
433 "MEDIAN": "Correct using the median of the good data",
434 },
435 )
437 # Overscan subtraction configuration.
438 doOverscan = pexConfig.Field(
439 dtype=bool,
440 doc="Do overscan subtraction?",
441 default=True,
442 )
443 overscan = pexConfig.ConfigurableField(
444 target=OverscanCorrectionTask,
445 doc="Overscan subtraction task for image segments.",
446 )
448 overscanFitType = pexConfig.ChoiceField(
449 dtype=str,
450 doc="The method for fitting the overscan bias level.",
451 default='MEDIAN',
452 allowed={
453 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
454 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
455 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
456 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
457 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
458 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
459 "MEAN": "Correct using the mean of the overscan region",
460 "MEANCLIP": "Correct using a clipped mean of the overscan region",
461 "MEDIAN": "Correct using the median of the overscan region",
462 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
463 },
464 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
465 " This option will no longer be used, and will be removed after v20.")
466 )
467 overscanOrder = pexConfig.Field(
468 dtype=int,
469 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, "
470 "or number of spline knots if overscan fit type is a spline."),
471 default=1,
472 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
473 " This option will no longer be used, and will be removed after v20.")
474 )
475 overscanNumSigmaClip = pexConfig.Field(
476 dtype=float,
477 doc="Rejection threshold (sigma) for collapsing overscan before fit",
478 default=3.0,
479 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
480 " This option will no longer be used, and will be removed after v20.")
481 )
482 overscanIsInt = pexConfig.Field(
483 dtype=bool,
484 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
485 " and overscan.FitType=MEDIAN_PER_ROW.",
486 default=True,
487 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
488 " This option will no longer be used, and will be removed after v20.")
489 )
490 # These options do not get deprecated, as they define how we slice up the image data.
491 overscanNumLeadingColumnsToSkip = pexConfig.Field(
492 dtype=int,
493 doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
494 default=0,
495 )
496 overscanNumTrailingColumnsToSkip = pexConfig.Field(
497 dtype=int,
498 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
499 default=0,
500 )
501 overscanMaxDev = pexConfig.Field( 501 ↛ exitline 501 didn't jump to the function exit
502 dtype=float,
503 doc="Maximum deviation from the median for overscan",
504 default=1000.0, check=lambda x: x > 0
505 )
506 overscanBiasJump = pexConfig.Field(
507 dtype=bool,
508 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
509 default=False,
510 )
511 overscanBiasJumpKeyword = pexConfig.Field(
512 dtype=str,
513 doc="Header keyword containing information about devices.",
514 default="NO_SUCH_KEY",
515 )
516 overscanBiasJumpDevices = pexConfig.ListField(
517 dtype=str,
518 doc="List of devices that need piecewise overscan correction.",
519 default=(),
520 )
521 overscanBiasJumpLocation = pexConfig.Field(
522 dtype=int,
523 doc="Location of bias jump along y-axis.",
524 default=0,
525 )
527 # Amplifier to CCD assembly configuration
528 doAssembleCcd = pexConfig.Field(
529 dtype=bool,
530 default=True,
531 doc="Assemble amp-level exposures into a ccd-level exposure?"
532 )
533 assembleCcd = pexConfig.ConfigurableField(
534 target=AssembleCcdTask,
535 doc="CCD assembly task",
536 )
538 # General calibration configuration.
539 doAssembleIsrExposures = pexConfig.Field(
540 dtype=bool,
541 default=False,
542 doc="Assemble amp-level calibration exposures into ccd-level exposure?"
543 )
544 doTrimToMatchCalib = pexConfig.Field(
545 dtype=bool,
546 default=False,
547 doc="Trim raw data to match calibration bounding boxes?"
548 )
550 # Bias subtraction.
551 doBias = pexConfig.Field(
552 dtype=bool,
553 doc="Apply bias frame correction?",
554 default=True,
555 )
556 biasDataProductName = pexConfig.Field(
557 dtype=str,
558 doc="Name of the bias data product",
559 default="bias",
560 )
561 doBiasBeforeOverscan = pexConfig.Field(
562 dtype=bool,
563 doc="Reverse order of overscan and bias correction.",
564 default=False
565 )
567 # Variance construction
568 doVariance = pexConfig.Field(
569 dtype=bool,
570 doc="Calculate variance?",
571 default=True
572 )
573 gain = pexConfig.Field(
574 dtype=float,
575 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
576 default=float("NaN"),
577 )
578 readNoise = pexConfig.Field(
579 dtype=float,
580 doc="The read noise to use if no Detector is present in the Exposure",
581 default=0.0,
582 )
583 doEmpiricalReadNoise = pexConfig.Field(
584 dtype=bool,
585 default=False,
586 doc="Calculate empirical read noise instead of value from AmpInfo data?"
587 )
588 usePtcReadNoise = pexConfig.Field(
589 dtype=bool,
590 default=False,
591 doc="Use readnoise values from the Photon Transfer Curve?"
592 )
593 # Linearization.
594 doLinearize = pexConfig.Field(
595 dtype=bool,
596 doc="Correct for nonlinearity of the detector's response?",
597 default=True,
598 )
600 # Crosstalk.
601 doCrosstalk = pexConfig.Field(
602 dtype=bool,
603 doc="Apply intra-CCD crosstalk correction?",
604 default=False,
605 )
606 doCrosstalkBeforeAssemble = pexConfig.Field(
607 dtype=bool,
608 doc="Apply crosstalk correction before CCD assembly, and before trimming?",
609 default=False,
610 )
611 crosstalk = pexConfig.ConfigurableField(
612 target=CrosstalkTask,
613 doc="Intra-CCD crosstalk correction",
614 )
616 # Masking options.
617 doDefect = pexConfig.Field(
618 dtype=bool,
619 doc="Apply correction for CCD defects, e.g. hot pixels?",
620 default=True,
621 )
622 doNanMasking = pexConfig.Field(
623 dtype=bool,
624 doc="Mask non-finite (NAN, inf) pixels?",
625 default=True,
626 )
627 doWidenSaturationTrails = pexConfig.Field(
628 dtype=bool,
629 doc="Widen bleed trails based on their width?",
630 default=True
631 )
633 # Brighter-Fatter correction.
634 doBrighterFatter = pexConfig.Field(
635 dtype=bool,
636 default=False,
637 doc="Apply the brighter-fatter correction?"
638 )
639 brighterFatterLevel = pexConfig.ChoiceField(
640 dtype=str,
641 default="DETECTOR",
642 doc="The level at which to correct for brighter-fatter.",
643 allowed={
644 "AMP": "Every amplifier treated separately.",
645 "DETECTOR": "One kernel per detector",
646 }
647 )
648 brighterFatterMaxIter = pexConfig.Field(
649 dtype=int,
650 default=10,
651 doc="Maximum number of iterations for the brighter-fatter correction"
652 )
653 brighterFatterThreshold = pexConfig.Field(
654 dtype=float,
655 default=1000,
656 doc="Threshold used to stop iterating the brighter-fatter correction. It is the "
657 "absolute value of the difference between the current corrected image and the one "
658 "from the previous iteration summed over all the pixels."
659 )
660 brighterFatterApplyGain = pexConfig.Field(
661 dtype=bool,
662 default=True,
663 doc="Should the gain be applied when applying the brighter-fatter correction?"
664 )
665 brighterFatterMaskListToInterpolate = pexConfig.ListField(
666 dtype=str,
667 doc="List of mask planes that should be interpolated over when applying the brighter-fatter "
668 "correction.",
669 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"],
670 )
671 brighterFatterMaskGrowSize = pexConfig.Field(
672 dtype=int,
673 default=0,
674 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
675 "when brighter-fatter correction is applied."
676 )
678 # Dark subtraction.
679 doDark = pexConfig.Field(
680 dtype=bool,
681 doc="Apply dark frame correction?",
682 default=True,
683 )
684 darkDataProductName = pexConfig.Field(
685 dtype=str,
686 doc="Name of the dark data product",
687 default="dark",
688 )
690 # Camera-specific stray light removal.
691 doStrayLight = pexConfig.Field(
692 dtype=bool,
693 doc="Subtract stray light in the y-band (due to encoder LEDs)?",
694 default=False,
695 )
696 strayLight = pexConfig.ConfigurableField(
697 target=StrayLightTask,
698 doc="y-band stray light correction"
699 )
701 # Flat correction.
702 doFlat = pexConfig.Field(
703 dtype=bool,
704 doc="Apply flat field correction?",
705 default=True,
706 )
707 flatDataProductName = pexConfig.Field(
708 dtype=str,
709 doc="Name of the flat data product",
710 default="flat",
711 )
712 flatScalingType = pexConfig.ChoiceField(
713 dtype=str,
714 doc="The method for scaling the flat on the fly.",
715 default='USER',
716 allowed={
717 "USER": "Scale by flatUserScale",
718 "MEAN": "Scale by the inverse of the mean",
719 "MEDIAN": "Scale by the inverse of the median",
720 },
721 )
722 flatUserScale = pexConfig.Field(
723 dtype=float,
724 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
725 default=1.0,
726 )
727 doTweakFlat = pexConfig.Field(
728 dtype=bool,
729 doc="Tweak flats to match observed amplifier ratios?",
730 default=False
731 )
733 # Amplifier normalization based on gains instead of using flats configuration.
734 doApplyGains = pexConfig.Field(
735 dtype=bool,
736 doc="Correct the amplifiers for their gains instead of applying flat correction",
737 default=False,
738 )
739 usePtcGains = pexConfig.Field(
740 dtype=bool,
741 doc="Use the gain values from the Photon Transfer Curve?",
742 default=False,
743 )
744 normalizeGains = pexConfig.Field(
745 dtype=bool,
746 doc="Normalize all the amplifiers in each CCD to have the same median value.",
747 default=False,
748 )
750 # Fringe correction.
751 doFringe = pexConfig.Field(
752 dtype=bool,
753 doc="Apply fringe correction?",
754 default=True,
755 )
756 fringe = pexConfig.ConfigurableField(
757 target=FringeTask,
758 doc="Fringe subtraction task",
759 )
760 fringeAfterFlat = pexConfig.Field(
761 dtype=bool,
762 doc="Do fringe subtraction after flat-fielding?",
763 default=True,
764 )
766 # Initial CCD-level background statistics options.
767 doMeasureBackground = pexConfig.Field(
768 dtype=bool,
769 doc="Measure the background level on the reduced image?",
770 default=False,
771 )
773 # Camera-specific masking configuration.
774 doCameraSpecificMasking = pexConfig.Field(
775 dtype=bool,
776 doc="Mask camera-specific bad regions?",
777 default=False,
778 )
779 masking = pexConfig.ConfigurableField(
780 target=MaskingTask,
781 doc="Masking task."
782 )
784 # Interpolation options.
786 doInterpolate = pexConfig.Field(
787 dtype=bool,
788 doc="Interpolate masked pixels?",
789 default=True,
790 )
791 doSaturationInterpolation = pexConfig.Field(
792 dtype=bool,
793 doc="Perform interpolation over pixels masked as saturated?"
794 " NB: This is independent of doSaturation; if that is False this plane"
795 " will likely be blank, resulting in a no-op here.",
796 default=True,
797 )
798 doNanInterpolation = pexConfig.Field(
799 dtype=bool,
800 doc="Perform interpolation over pixels masked as NaN?"
801 " NB: This is independent of doNanMasking; if that is False this plane"
802 " will likely be blank, resulting in a no-op here.",
803 default=True,
804 )
805 doNanInterpAfterFlat = pexConfig.Field(
806 dtype=bool,
807 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
808 "also have to interpolate them before flat-fielding."),
809 default=False,
810 )
811 maskListToInterpolate = pexConfig.ListField(
812 dtype=str,
813 doc="List of mask planes that should be interpolated.",
814 default=['SAT', 'BAD'],
815 )
816 doSaveInterpPixels = pexConfig.Field(
817 dtype=bool,
818 doc="Save a copy of the pre-interpolated pixel values?",
819 default=False,
820 )
822 # Default photometric calibration options.
823 fluxMag0T1 = pexConfig.DictField(
824 keytype=str,
825 itemtype=float,
826 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
827 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
828 ))
829 )
830 defaultFluxMag0T1 = pexConfig.Field(
831 dtype=float,
832 doc="Default value for fluxMag0T1 (for an unrecognized filter).",
833 default=pow(10.0, 0.4*28.0)
834 )
836 # Vignette correction configuration.
837 doVignette = pexConfig.Field(
838 dtype=bool,
839 doc="Apply vignetting parameters?",
840 default=False,
841 )
842 vignette = pexConfig.ConfigurableField(
843 target=VignetteTask,
844 doc="Vignetting task.",
845 )
847 # Transmission curve configuration.
848 doAttachTransmissionCurve = pexConfig.Field(
849 dtype=bool,
850 default=False,
851 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
852 )
853 doUseOpticsTransmission = pexConfig.Field(
854 dtype=bool,
855 default=True,
856 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
857 )
858 doUseFilterTransmission = pexConfig.Field(
859 dtype=bool,
860 default=True,
861 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
862 )
863 doUseSensorTransmission = pexConfig.Field(
864 dtype=bool,
865 default=True,
866 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
867 )
868 doUseAtmosphereTransmission = pexConfig.Field(
869 dtype=bool,
870 default=True,
871 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
872 )
874 # Illumination correction.
875 doIlluminationCorrection = pexConfig.Field(
876 dtype=bool,
877 default=False,
878 doc="Perform illumination correction?"
879 )
880 illuminationCorrectionDataProductName = pexConfig.Field(
881 dtype=str,
882 doc="Name of the illumination correction data product.",
883 default="illumcor",
884 )
885 illumScale = pexConfig.Field(
886 dtype=float,
887 doc="Scale factor for the illumination correction.",
888 default=1.0,
889 )
890 illumFilters = pexConfig.ListField(
891 dtype=str,
892 default=[],
893 doc="Only perform illumination correction for these filters."
894 )
896 # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
897 doWrite = pexConfig.Field(
898 dtype=bool,
899 doc="Persist postISRCCD?",
900 default=True,
901 )
903 def validate(self):
904 super().validate()
905 if self.doFlat and self.doApplyGains:
906 raise ValueError("You may not specify both doFlat and doApplyGains")
907 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib:
908 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
909 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate:
910 self.maskListToInterpolate.append(self.saturatedMaskName)
911 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate:
912 self.maskListToInterpolate.remove(self.saturatedMaskName)
913 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
914 self.maskListToInterpolate.append("UNMASKEDNAN")
917class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
918 """Apply common instrument signature correction algorithms to a raw frame.
920 The process for correcting imaging data is very similar from
921 camera to camera. This task provides a vanilla implementation of
922 doing these corrections, including the ability to turn certain
923 corrections off if they are not needed. The inputs to the primary
924 method, `run()`, are a raw exposure to be corrected and the
925 calibration data products. The raw input is a single chip sized
926 mosaic of all amps including overscans and other non-science
927 pixels. The method `runDataRef()` identifies and defines the
928 calibration data products, and is intended for use by a
929 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
930 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
931 subclassed for different camera, although the most camera specific
932 methods have been split into subtasks that can be redirected
933 appropriately.
935 The __init__ method sets up the subtasks for ISR processing, using
936 the defaults from `lsst.ip.isr`.
938 Parameters
939 ----------
940 args : `list`
941 Positional arguments passed to the Task constructor. None used at this time.
942 kwargs : `dict`, optional
943 Keyword arguments passed on to the Task constructor. None used at this time.
944 """
945 ConfigClass = IsrTaskConfig
946 _DefaultName = "isr"
948 def __init__(self, **kwargs):
949 super().__init__(**kwargs)
950 self.makeSubtask("assembleCcd")
951 self.makeSubtask("crosstalk")
952 self.makeSubtask("strayLight")
953 self.makeSubtask("fringe")
954 self.makeSubtask("masking")
955 self.makeSubtask("overscan")
956 self.makeSubtask("vignette")
958 def runQuantum(self, butlerQC, inputRefs, outputRefs):
959 inputs = butlerQC.get(inputRefs)
961 try:
962 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
963 except Exception as e:
964 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
965 (inputRefs, e))
967 inputs['isGen3'] = True
969 detector = inputs['ccdExposure'].getDetector()
971 if self.config.doCrosstalk is True:
972 # Crosstalk sources need to be defined by the pipeline
973 # yaml if they exist.
974 if 'crosstalk' in inputs and inputs['crosstalk'] is not None:
975 if not isinstance(inputs['crosstalk'], CrosstalkCalib):
976 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk'])
977 else:
978 coeffVector = (self.config.crosstalk.crosstalkValues
979 if self.config.crosstalk.useConfigCoefficients else None)
980 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
981 inputs['crosstalk'] = crosstalkCalib
982 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0:
983 if 'crosstalkSources' not in inputs:
984 self.log.warn("No crosstalkSources found for chip with interChip terms!")
986 if self.doLinearize(detector) is True:
987 if 'linearizer' in inputs:
988 if isinstance(inputs['linearizer'], dict):
989 linearizer = linearize.Linearizer(detector=detector, log=self.log)
990 linearizer.fromYaml(inputs['linearizer'])
991 self.log.warn("Dictionary linearizers will be deprecated in DM-28741.")
992 elif isinstance(inputs['linearizer'], numpy.ndarray):
993 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None),
994 detector=detector,
995 log=self.log)
996 self.log.warn("Bare lookup table linearizers will be deprecated in DM-28741.")
997 else:
998 linearizer = inputs['linearizer']
999 linearizer.log = self.log
1000 inputs['linearizer'] = linearizer
1001 else:
1002 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log)
1003 self.log.warn("Constructing linearizer from cameraGeom information.")
1005 if self.config.doDefect is True:
1006 if "defects" in inputs and inputs['defects'] is not None:
1007 # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
1008 # masking expects a list of defects defined by their bounding box
1009 if not isinstance(inputs["defects"], Defects):
1010 inputs["defects"] = Defects.fromTable(inputs["defects"])
1012 # Load the correct style of brighter-fatter kernel, and repack
1013 # the information as a numpy array.
1014 if self.config.doBrighterFatter:
1015 brighterFatterKernel = inputs.pop('newBFKernel', None)
1016 if brighterFatterKernel is None:
1017 brighterFatterKernel = inputs.get('bfKernel', None)
1019 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1020 # This is a ISR calib kernel
1021 detName = detector.getName()
1022 level = brighterFatterKernel.level
1024 # This is expected to be a dictionary of amp-wise gains.
1025 inputs['bfGains'] = brighterFatterKernel.gain
1026 if self.config.brighterFatterLevel == 'DETECTOR':
1027 if level == 'DETECTOR':
1028 if detName in brighterFatterKernel.detKernels:
1029 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1030 else:
1031 raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1032 elif level == 'AMP':
1033 self.log.warn("Making DETECTOR level kernel from AMP based brighter fatter kernels.")
1034 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1035 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1036 elif self.config.brighterFatterLevel == 'AMP':
1037 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1039 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
1040 expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
1041 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
1042 expId=expId,
1043 assembler=self.assembleCcd
1044 if self.config.doAssembleIsrExposures else None)
1045 else:
1046 inputs['fringes'] = pipeBase.Struct(fringes=None)
1048 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
1049 if 'strayLightData' not in inputs:
1050 inputs['strayLightData'] = None
1052 outputs = self.run(**inputs)
1053 butlerQC.put(outputs, outputRefs)
1055 def readIsrData(self, dataRef, rawExposure):
1056 """Retrieve necessary frames for instrument signature removal.
1058 Pre-fetching all required ISR data products limits the IO
1059 required by the ISR. Any conflict between the calibration data
1060 available and that needed for ISR is also detected prior to
1061 doing processing, allowing it to fail quickly.
1063 Parameters
1064 ----------
1065 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1066 Butler reference of the detector data to be processed
1067 rawExposure : `afw.image.Exposure`
1068 The raw exposure that will later be corrected with the
1069 retrieved calibration data; should not be modified in this
1070 method.
1072 Returns
1073 -------
1074 result : `lsst.pipe.base.Struct`
1075 Result struct with components (which may be `None`):
1076 - ``bias``: bias calibration frame (`afw.image.Exposure`)
1077 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
1078 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1079 - ``dark``: dark calibration frame (`afw.image.Exposure`)
1080 - ``flat``: flat calibration frame (`afw.image.Exposure`)
1081 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1082 - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1083 - ``fringes``: `lsst.pipe.base.Struct` with components:
1084 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1085 - ``seed``: random seed derived from the ccdExposureId for random
1086 number generator (`uint32`).
1087 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1088 A ``TransmissionCurve`` that represents the throughput of the optics,
1089 to be evaluated in focal-plane coordinates.
1090 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1091 A ``TransmissionCurve`` that represents the throughput of the filter
1092 itself, to be evaluated in focal-plane coordinates.
1093 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1094 A ``TransmissionCurve`` that represents the throughput of the sensor
1095 itself, to be evaluated in post-assembly trimmed detector coordinates.
1096 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1097 A ``TransmissionCurve`` that represents the throughput of the
1098 atmosphere, assumed to be spatially constant.
1099 - ``strayLightData`` : `object`
1100 An opaque object containing calibration information for
1101 stray-light correction. If `None`, no correction will be
1102 performed.
1103 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1105 Raises
1106 ------
1107 NotImplementedError :
1108 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1109 """
1110 try:
1111 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1112 dateObs = dateObs.toPython().isoformat()
1113 except RuntimeError:
1114 self.log.warn("Unable to identify dateObs for rawExposure.")
1115 dateObs = None
1117 ccd = rawExposure.getDetector()
1118 filterLabel = rawExposure.getFilterLabel()
1119 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1120 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
1121 biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
1122 if self.config.doBias else None)
1123 # immediate=True required for functors and linearizers are functors; see ticket DM-6515
1124 linearizer = (dataRef.get("linearizer", immediate=True)
1125 if self.doLinearize(ccd) else None)
1126 if linearizer is not None and not isinstance(linearizer, numpy.ndarray):
1127 linearizer.log = self.log
1128 if isinstance(linearizer, numpy.ndarray):
1129 linearizer = linearize.Linearizer(table=linearizer, detector=ccd)
1131 crosstalkCalib = None
1132 if self.config.doCrosstalk:
1133 try:
1134 crosstalkCalib = dataRef.get("crosstalk", immediate=True)
1135 except NoResults:
1136 coeffVector = (self.config.crosstalk.crosstalkValues
1137 if self.config.crosstalk.useConfigCoefficients else None)
1138 crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1139 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1140 if self.config.doCrosstalk else None)
1142 darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
1143 if self.config.doDark else None)
1144 flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName,
1145 dateObs=dateObs)
1146 if self.config.doFlat else None)
1148 brighterFatterKernel = None
1149 brighterFatterGains = None
1150 if self.config.doBrighterFatter is True:
1151 try:
1152 # Use the new-style cp_pipe version of the kernel if it exists
1153 # If using a new-style kernel, always use the self-consistent
1154 # gains, i.e. the ones inside the kernel object itself
1155 brighterFatterKernel = dataRef.get("brighterFatterKernel")
1156 brighterFatterGains = brighterFatterKernel.gain
1157 self.log.info("New style brighter-fatter kernel (brighterFatterKernel) loaded")
1158 except NoResults:
1159 try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
1160 brighterFatterKernel = dataRef.get("bfKernel")
1161 self.log.info("Old style brighter-fatter kernel (bfKernel) loaded")
1162 except NoResults:
1163 brighterFatterKernel = None
1164 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1165 # If the kernel is not an ndarray, it's the cp_pipe version
1166 # so extract the kernel for this detector, or raise an error
1167 if self.config.brighterFatterLevel == 'DETECTOR':
1168 if brighterFatterKernel.detKernels:
1169 brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()]
1170 else:
1171 raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1172 else:
1173 # TODO DM-15631 for implementing this
1174 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1176 defectList = (dataRef.get("defects")
1177 if self.config.doDefect else None)
1178 expId = rawExposure.getInfo().getVisitInfo().getExposureId()
1179 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1180 if self.config.doAssembleIsrExposures else None)
1181 if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1182 else pipeBase.Struct(fringes=None))
1184 if self.config.doAttachTransmissionCurve:
1185 opticsTransmission = (dataRef.get("transmission_optics")
1186 if self.config.doUseOpticsTransmission else None)
1187 filterTransmission = (dataRef.get("transmission_filter")
1188 if self.config.doUseFilterTransmission else None)
1189 sensorTransmission = (dataRef.get("transmission_sensor")
1190 if self.config.doUseSensorTransmission else None)
1191 atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1192 if self.config.doUseAtmosphereTransmission else None)
1193 else:
1194 opticsTransmission = None
1195 filterTransmission = None
1196 sensorTransmission = None
1197 atmosphereTransmission = None
1199 if self.config.doStrayLight:
1200 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1201 else:
1202 strayLightData = None
1204 illumMaskedImage = (self.getIsrExposure(dataRef,
1205 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1206 if (self.config.doIlluminationCorrection
1207 and physicalFilter in self.config.illumFilters)
1208 else None)
1210 # Struct should include only kwargs to run()
1211 return pipeBase.Struct(bias=biasExposure,
1212 linearizer=linearizer,
1213 crosstalk=crosstalkCalib,
1214 crosstalkSources=crosstalkSources,
1215 dark=darkExposure,
1216 flat=flatExposure,
1217 bfKernel=brighterFatterKernel,
1218 bfGains=brighterFatterGains,
1219 defects=defectList,
1220 fringes=fringeStruct,
1221 opticsTransmission=opticsTransmission,
1222 filterTransmission=filterTransmission,
1223 sensorTransmission=sensorTransmission,
1224 atmosphereTransmission=atmosphereTransmission,
1225 strayLightData=strayLightData,
1226 illumMaskedImage=illumMaskedImage
1227 )
1229 @pipeBase.timeMethod
1230 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1231 crosstalk=None, crosstalkSources=None,
1232 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1233 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1234 sensorTransmission=None, atmosphereTransmission=None,
1235 detectorNum=None, strayLightData=None, illumMaskedImage=None,
1236 isGen3=False,
1237 ):
1238 """Perform instrument signature removal on an exposure.
1240 Steps included in the ISR processing, in order performed, are:
1241 - saturation and suspect pixel masking
1242 - overscan subtraction
1243 - CCD assembly of individual amplifiers
1244 - bias subtraction
1245 - variance image construction
1246 - linearization of non-linear response
1247 - crosstalk masking
1248 - brighter-fatter correction
1249 - dark subtraction
1250 - fringe correction
1251 - stray light subtraction
1252 - flat correction
1253 - masking of known defects and camera specific features
1254 - vignette calculation
1255 - appending transmission curve and distortion model
1257 Parameters
1258 ----------
1259 ccdExposure : `lsst.afw.image.Exposure`
1260 The raw exposure that is to be run through ISR. The
1261 exposure is modified by this method.
1262 camera : `lsst.afw.cameraGeom.Camera`, optional
1263 The camera geometry for this exposure. Required if ``isGen3`` is
1264 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1265 ``flat`` does not have an associated detector.
1266 bias : `lsst.afw.image.Exposure`, optional
1267 Bias calibration frame.
1268 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1269 Functor for linearization.
1270 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1271 Calibration for crosstalk.
1272 crosstalkSources : `list`, optional
1273 List of possible crosstalk sources.
1274 dark : `lsst.afw.image.Exposure`, optional
1275 Dark calibration frame.
1276 flat : `lsst.afw.image.Exposure`, optional
1277 Flat calibration frame.
1278 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
1279 Photon transfer curve dataset, with, e.g., gains
1280 and read noise.
1281 bfKernel : `numpy.ndarray`, optional
1282 Brighter-fatter kernel.
1283 bfGains : `dict` of `float`, optional
1284 Gains used to override the detector's nominal gains for the
1285 brighter-fatter correction. A dict keyed by amplifier name for
1286 the detector in question.
1287 defects : `lsst.ip.isr.Defects`, optional
1288 List of defects.
1289 fringes : `lsst.pipe.base.Struct`, optional
1290 Struct containing the fringe correction data, with
1291 elements:
1292 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1293 - ``seed``: random seed derived from the ccdExposureId for random
1294 number generator (`uint32`)
1295 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1296 A ``TransmissionCurve`` that represents the throughput of the optics,
1297 to be evaluated in focal-plane coordinates.
1298 filterTransmission : `lsst.afw.image.TransmissionCurve`
1299 A ``TransmissionCurve`` that represents the throughput of the filter
1300 itself, to be evaluated in focal-plane coordinates.
1301 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1302 A ``TransmissionCurve`` that represents the throughput of the sensor
1303 itself, to be evaluated in post-assembly trimmed detector coordinates.
1304 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1305 A ``TransmissionCurve`` that represents the throughput of the
1306 atmosphere, assumed to be spatially constant.
1307 detectorNum : `int`, optional
1308 The integer number for the detector to process.
1309 isGen3 : bool, optional
1310 Flag this call to run() as using the Gen3 butler environment.
1311 strayLightData : `object`, optional
1312 Opaque object containing calibration information for stray-light
1313 correction. If `None`, no correction will be performed.
1314 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1315 Illumination correction image.
1317 Returns
1318 -------
1319 result : `lsst.pipe.base.Struct`
1320 Result struct with component:
1321 - ``exposure`` : `afw.image.Exposure`
1322 The fully ISR corrected exposure.
1323 - ``outputExposure`` : `afw.image.Exposure`
1324 An alias for `exposure`
1325 - ``ossThumb`` : `numpy.ndarray`
1326 Thumbnail image of the exposure after overscan subtraction.
1327 - ``flattenedThumb`` : `numpy.ndarray`
1328 Thumbnail image of the exposure after flat-field correction.
1330 Raises
1331 ------
1332 RuntimeError
1333 Raised if a configuration option is set to True, but the
1334 required calibration data has not been specified.
1336 Notes
1337 -----
1338 The current processed exposure can be viewed by setting the
1339 appropriate lsstDebug entries in the `debug.display`
1340 dictionary. The names of these entries correspond to some of
1341 the IsrTaskConfig Boolean options, with the value denoting the
1342 frame to use. The exposure is shown inside the matching
1343 option check and after the processing of that step has
1344 finished. The steps with debug points are:
1346 doAssembleCcd
1347 doBias
1348 doCrosstalk
1349 doBrighterFatter
1350 doDark
1351 doFringe
1352 doStrayLight
1353 doFlat
1355 In addition, setting the "postISRCCD" entry displays the
1356 exposure after all ISR processing has finished.
1358 """
1360 if isGen3 is True:
1361 # Gen3 currently cannot automatically do configuration overrides.
1362 # DM-15257 looks to discuss this issue.
1363 # Configure input exposures;
1364 if detectorNum is None:
1365 raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1367 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1368 bias = self.ensureExposure(bias, camera, detectorNum)
1369 dark = self.ensureExposure(dark, camera, detectorNum)
1370 flat = self.ensureExposure(flat, camera, detectorNum)
1371 else:
1372 if isinstance(ccdExposure, ButlerDataRef):
1373 return self.runDataRef(ccdExposure)
1375 ccd = ccdExposure.getDetector()
1376 filterLabel = ccdExposure.getFilterLabel()
1377 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1379 if not ccd:
1380 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1381 ccd = [FakeAmp(ccdExposure, self.config)]
1383 # Validate Input
1384 if self.config.doBias and bias is None:
1385 raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1386 if self.doLinearize(ccd) and linearizer is None:
1387 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1388 if self.config.doBrighterFatter and bfKernel is None:
1389 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1390 if self.config.doDark and dark is None:
1391 raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1392 if self.config.doFlat and flat is None:
1393 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1394 if self.config.doDefect and defects is None:
1395 raise RuntimeError("Must supply defects if config.doDefect=True.")
1396 if (self.config.doFringe and physicalFilter in self.fringe.config.filters
1397 and fringes.fringes is None):
1398 # The `fringes` object needs to be a pipeBase.Struct, as
1399 # we use it as a `dict` for the parameters of
1400 # `FringeTask.run()`. The `fringes.fringes` `list` may
1401 # not be `None` if `doFringe=True`. Otherwise, raise.
1402 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1403 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters
1404 and illumMaskedImage is None):
1405 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1407 # Begin ISR processing.
1408 if self.config.doConvertIntToFloat:
1409 self.log.info("Converting exposure to floating point values.")
1410 ccdExposure = self.convertIntToFloat(ccdExposure)
1412 if self.config.doBias and self.config.doBiasBeforeOverscan:
1413 self.log.info("Applying bias correction.")
1414 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1415 trimToFit=self.config.doTrimToMatchCalib)
1416 self.debugView(ccdExposure, "doBias")
1418 # Amplifier level processing.
1419 overscans = []
1420 for amp in ccd:
1421 # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1422 if ccdExposure.getBBox().contains(amp.getBBox()):
1423 # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1424 badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1426 if self.config.doOverscan and not badAmp:
1427 # Overscan correction on amp-by-amp basis.
1428 overscanResults = self.overscanCorrection(ccdExposure, amp)
1429 self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1430 if overscanResults is not None and \
1431 self.config.qa is not None and self.config.qa.saveStats is True:
1432 if isinstance(overscanResults.overscanFit, float):
1433 qaMedian = overscanResults.overscanFit
1434 qaStdev = float("NaN")
1435 else:
1436 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1437 afwMath.MEDIAN | afwMath.STDEVCLIP)
1438 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1439 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1441 self.metadata.set(f"FIT MEDIAN {amp.getName()}", qaMedian)
1442 self.metadata.set(f"FIT STDEV {amp.getName()}", qaStdev)
1443 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1444 amp.getName(), qaMedian, qaStdev)
1446 # Residuals after overscan correction
1447 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1448 afwMath.MEDIAN | afwMath.STDEVCLIP)
1449 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1450 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1452 self.metadata.set(f"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter)
1453 self.metadata.set(f"RESIDUAL STDEV {amp.getName()}", qaStdevAfter)
1454 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f",
1455 amp.getName(), qaMedianAfter, qaStdevAfter)
1457 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1458 else:
1459 if badAmp:
1460 self.log.warn("Amplifier %s is bad.", amp.getName())
1461 overscanResults = None
1463 overscans.append(overscanResults if overscanResults is not None else None)
1464 else:
1465 self.log.info("Skipped OSCAN for %s.", amp.getName())
1467 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1468 self.log.info("Applying crosstalk correction.")
1469 self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1470 crosstalkSources=crosstalkSources, camera=camera)
1471 self.debugView(ccdExposure, "doCrosstalk")
1473 if self.config.doAssembleCcd:
1474 self.log.info("Assembling CCD from amplifiers.")
1475 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1477 if self.config.expectWcs and not ccdExposure.getWcs():
1478 self.log.warn("No WCS found in input exposure.")
1479 self.debugView(ccdExposure, "doAssembleCcd")
1481 ossThumb = None
1482 if self.config.qa.doThumbnailOss:
1483 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1485 if self.config.doBias and not self.config.doBiasBeforeOverscan:
1486 self.log.info("Applying bias correction.")
1487 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1488 trimToFit=self.config.doTrimToMatchCalib)
1489 self.debugView(ccdExposure, "doBias")
1491 if self.config.doVariance:
1492 for amp, overscanResults in zip(ccd, overscans):
1493 if ccdExposure.getBBox().contains(amp.getBBox()):
1494 self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1495 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1496 if overscanResults is not None:
1497 self.updateVariance(ampExposure, amp,
1498 overscanImage=overscanResults.overscanImage,
1499 ptcDataset=ptc)
1500 else:
1501 self.updateVariance(ampExposure, amp,
1502 overscanImage=None,
1503 ptcDataset=ptc)
1504 if self.config.qa is not None and self.config.qa.saveStats is True:
1505 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1506 afwMath.MEDIAN | afwMath.STDEVCLIP)
1507 self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1508 qaStats.getValue(afwMath.MEDIAN))
1509 self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1510 qaStats.getValue(afwMath.STDEVCLIP))
1511 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1512 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1513 qaStats.getValue(afwMath.STDEVCLIP))
1515 if self.doLinearize(ccd):
1516 self.log.info("Applying linearizer.")
1517 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1518 detector=ccd, log=self.log)
1520 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1521 self.log.info("Applying crosstalk correction.")
1522 self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1523 crosstalkSources=crosstalkSources, isTrimmed=True)
1524 self.debugView(ccdExposure, "doCrosstalk")
1526 # Masking block. Optionally mask known defects, NAN/inf pixels, widen trails, and do
1527 # anything else the camera needs. Saturated and suspect pixels have already been masked.
1528 if self.config.doDefect:
1529 self.log.info("Masking defects.")
1530 self.maskDefect(ccdExposure, defects)
1532 if self.config.numEdgeSuspect > 0:
1533 self.log.info("Masking edges as SUSPECT.")
1534 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1535 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
1537 if self.config.doNanMasking:
1538 self.log.info("Masking non-finite (NAN, inf) value pixels.")
1539 self.maskNan(ccdExposure)
1541 if self.config.doWidenSaturationTrails:
1542 self.log.info("Widening saturation trails.")
1543 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1545 if self.config.doCameraSpecificMasking:
1546 self.log.info("Masking regions for camera specific reasons.")
1547 self.masking.run(ccdExposure)
1549 if self.config.doBrighterFatter:
1550 # We need to apply flats and darks before we can interpolate, and we
1551 # need to interpolate before we do B-F, but we do B-F without the
1552 # flats and darks applied so we can work in units of electrons or holes.
1553 # This context manager applies and then removes the darks and flats.
1554 #
1555 # We also do not want to interpolate values here, so operate on temporary
1556 # images so we can apply only the BF-correction and roll back the
1557 # interpolation.
1558 interpExp = ccdExposure.clone()
1559 with self.flatContext(interpExp, flat, dark):
1560 isrFunctions.interpolateFromMask(
1561 maskedImage=interpExp.getMaskedImage(),
1562 fwhm=self.config.fwhm,
1563 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1564 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1565 )
1566 bfExp = interpExp.clone()
1568 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.",
1569 type(bfKernel), type(bfGains))
1570 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1571 self.config.brighterFatterMaxIter,
1572 self.config.brighterFatterThreshold,
1573 self.config.brighterFatterApplyGain,
1574 bfGains)
1575 if bfResults[1] == self.config.brighterFatterMaxIter:
1576 self.log.warn("Brighter-fatter correction did not converge, final difference %f.",
1577 bfResults[0])
1578 else:
1579 self.log.info("Finished brighter-fatter correction in %d iterations.",
1580 bfResults[1])
1581 image = ccdExposure.getMaskedImage().getImage()
1582 bfCorr = bfExp.getMaskedImage().getImage()
1583 bfCorr -= interpExp.getMaskedImage().getImage()
1584 image += bfCorr
1586 # Applying the brighter-fatter correction applies a
1587 # convolution to the science image. At the edges this
1588 # convolution may not have sufficient valid pixels to
1589 # produce a valid correction. Mark pixels within the size
1590 # of the brighter-fatter kernel as EDGE to warn of this
1591 # fact.
1592 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1593 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1594 maskPlane="EDGE")
1596 if self.config.brighterFatterMaskGrowSize > 0:
1597 self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1598 for maskPlane in self.config.brighterFatterMaskListToInterpolate:
1599 isrFunctions.growMasks(ccdExposure.getMask(),
1600 radius=self.config.brighterFatterMaskGrowSize,
1601 maskNameList=maskPlane,
1602 maskValue=maskPlane)
1604 self.debugView(ccdExposure, "doBrighterFatter")
1606 if self.config.doDark:
1607 self.log.info("Applying dark correction.")
1608 self.darkCorrection(ccdExposure, dark)
1609 self.debugView(ccdExposure, "doDark")
1611 if self.config.doFringe and not self.config.fringeAfterFlat:
1612 self.log.info("Applying fringe correction before flat.")
1613 self.fringe.run(ccdExposure, **fringes.getDict())
1614 self.debugView(ccdExposure, "doFringe")
1616 if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1617 self.log.info("Checking strayLight correction.")
1618 self.strayLight.run(ccdExposure, strayLightData)
1619 self.debugView(ccdExposure, "doStrayLight")
1621 if self.config.doFlat:
1622 self.log.info("Applying flat correction.")
1623 self.flatCorrection(ccdExposure, flat)
1624 self.debugView(ccdExposure, "doFlat")
1626 if self.config.doApplyGains:
1627 self.log.info("Applying gain correction instead of flat.")
1628 if self.config.usePtcGains:
1629 self.log.info("Using gains from the Photon Transfer Curve.")
1630 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1631 ptcGains=ptc.gain)
1632 else:
1633 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1635 if self.config.doFringe and self.config.fringeAfterFlat:
1636 self.log.info("Applying fringe correction after flat.")
1637 self.fringe.run(ccdExposure, **fringes.getDict())
1639 if self.config.doVignette:
1640 self.log.info("Constructing Vignette polygon.")
1641 self.vignettePolygon = self.vignette.run(ccdExposure)
1643 if self.config.vignette.doWriteVignettePolygon:
1644 self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1646 if self.config.doAttachTransmissionCurve:
1647 self.log.info("Adding transmission curves.")
1648 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1649 filterTransmission=filterTransmission,
1650 sensorTransmission=sensorTransmission,
1651 atmosphereTransmission=atmosphereTransmission)
1653 flattenedThumb = None
1654 if self.config.qa.doThumbnailFlattened:
1655 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1657 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters:
1658 self.log.info("Performing illumination correction.")
1659 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1660 illumMaskedImage, illumScale=self.config.illumScale,
1661 trimToFit=self.config.doTrimToMatchCalib)
1663 preInterpExp = None
1664 if self.config.doSaveInterpPixels:
1665 preInterpExp = ccdExposure.clone()
1667 # Reset and interpolate bad pixels.
1668 #
1669 # Large contiguous bad regions (which should have the BAD mask
1670 # bit set) should have their values set to the image median.
1671 # This group should include defects and bad amplifiers. As the
1672 # area covered by these defects are large, there's little
1673 # reason to expect that interpolation would provide a more
1674 # useful value.
1675 #
1676 # Smaller defects can be safely interpolated after the larger
1677 # regions have had their pixel values reset. This ensures
1678 # that the remaining defects adjacent to bad amplifiers (as an
1679 # example) do not attempt to interpolate extreme values.
1680 if self.config.doSetBadRegions:
1681 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1682 if badPixelCount > 0:
1683 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1685 if self.config.doInterpolate:
1686 self.log.info("Interpolating masked pixels.")
1687 isrFunctions.interpolateFromMask(
1688 maskedImage=ccdExposure.getMaskedImage(),
1689 fwhm=self.config.fwhm,
1690 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1691 maskNameList=list(self.config.maskListToInterpolate)
1692 )
1694 self.roughZeroPoint(ccdExposure)
1696 if self.config.doMeasureBackground:
1697 self.log.info("Measuring background level.")
1698 self.measureBackground(ccdExposure, self.config.qa)
1700 if self.config.qa is not None and self.config.qa.saveStats is True:
1701 for amp in ccd:
1702 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1703 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1704 afwMath.MEDIAN | afwMath.STDEVCLIP)
1705 self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1706 qaStats.getValue(afwMath.MEDIAN))
1707 self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1708 qaStats.getValue(afwMath.STDEVCLIP))
1709 self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1710 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1711 qaStats.getValue(afwMath.STDEVCLIP))
1713 self.debugView(ccdExposure, "postISRCCD")
1715 return pipeBase.Struct(
1716 exposure=ccdExposure,
1717 ossThumb=ossThumb,
1718 flattenedThumb=flattenedThumb,
1720 preInterpExposure=preInterpExp,
1721 outputExposure=ccdExposure,
1722 outputOssThumbnail=ossThumb,
1723 outputFlattenedThumbnail=flattenedThumb,
1724 )
1726 @pipeBase.timeMethod
1727 def runDataRef(self, sensorRef):
1728 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1730 This method contains the `CmdLineTask` interface to the ISR
1731 processing. All IO is handled here, freeing the `run()` method
1732 to manage only pixel-level calculations. The steps performed
1733 are:
1734 - Read in necessary detrending/isr/calibration data.
1735 - Process raw exposure in `run()`.
1736 - Persist the ISR-corrected exposure as "postISRCCD" if
1737 config.doWrite=True.
1739 Parameters
1740 ----------
1741 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1742 DataRef of the detector data to be processed
1744 Returns
1745 -------
1746 result : `lsst.pipe.base.Struct`
1747 Result struct with component:
1748 - ``exposure`` : `afw.image.Exposure`
1749 The fully ISR corrected exposure.
1751 Raises
1752 ------
1753 RuntimeError
1754 Raised if a configuration option is set to True, but the
1755 required calibration data does not exist.
1757 """
1758 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1760 ccdExposure = sensorRef.get(self.config.datasetType)
1762 camera = sensorRef.get("camera")
1763 isrData = self.readIsrData(sensorRef, ccdExposure)
1765 result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1767 if self.config.doWrite:
1768 sensorRef.put(result.exposure, "postISRCCD")
1769 if result.preInterpExposure is not None:
1770 sensorRef.put(result.preInterpExposure, "postISRCCD_uninterpolated")
1771 if result.ossThumb is not None:
1772 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1773 if result.flattenedThumb is not None:
1774 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1776 return result
1778 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1779 """Retrieve a calibration dataset for removing instrument signature.
1781 Parameters
1782 ----------
1784 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1785 DataRef of the detector data to find calibration datasets
1786 for.
1787 datasetType : `str`
1788 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1789 dateObs : `str`, optional
1790 Date of the observation. Used to correct butler failures
1791 when using fallback filters.
1792 immediate : `Bool`
1793 If True, disable butler proxies to enable error handling
1794 within this routine.
1796 Returns
1797 -------
1798 exposure : `lsst.afw.image.Exposure`
1799 Requested calibration frame.
1801 Raises
1802 ------
1803 RuntimeError
1804 Raised if no matching calibration frame can be found.
1805 """
1806 try:
1807 exp = dataRef.get(datasetType, immediate=immediate)
1808 except Exception as exc1:
1809 if not self.config.fallbackFilterName:
1810 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1811 try:
1812 if self.config.useFallbackDate and dateObs:
1813 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1814 dateObs=dateObs, immediate=immediate)
1815 else:
1816 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1817 except Exception as exc2:
1818 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1819 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1820 self.log.warn("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1822 if self.config.doAssembleIsrExposures:
1823 exp = self.assembleCcd.assembleCcd(exp)
1824 return exp
1826 def ensureExposure(self, inputExp, camera, detectorNum):
1827 """Ensure that the data returned by Butler is a fully constructed exposure.
1829 ISR requires exposure-level image data for historical reasons, so if we did
1830 not recieve that from Butler, construct it from what we have, modifying the
1831 input in place.
1833 Parameters
1834 ----------
1835 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1836 `lsst.afw.image.ImageF`
1837 The input data structure obtained from Butler.
1838 camera : `lsst.afw.cameraGeom.camera`
1839 The camera associated with the image. Used to find the appropriate
1840 detector.
1841 detectorNum : `int`
1842 The detector this exposure should match.
1844 Returns
1845 -------
1846 inputExp : `lsst.afw.image.Exposure`
1847 The re-constructed exposure, with appropriate detector parameters.
1849 Raises
1850 ------
1851 TypeError
1852 Raised if the input data cannot be used to construct an exposure.
1853 """
1854 if isinstance(inputExp, afwImage.DecoratedImageU):
1855 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1856 elif isinstance(inputExp, afwImage.ImageF):
1857 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1858 elif isinstance(inputExp, afwImage.MaskedImageF):
1859 inputExp = afwImage.makeExposure(inputExp)
1860 elif isinstance(inputExp, afwImage.Exposure):
1861 pass
1862 elif inputExp is None:
1863 # Assume this will be caught by the setup if it is a problem.
1864 return inputExp
1865 else:
1866 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1867 (type(inputExp), ))
1869 if inputExp.getDetector() is None:
1870 inputExp.setDetector(camera[detectorNum])
1872 return inputExp
1874 def convertIntToFloat(self, exposure):
1875 """Convert exposure image from uint16 to float.
1877 If the exposure does not need to be converted, the input is
1878 immediately returned. For exposures that are converted to use
1879 floating point pixels, the variance is set to unity and the
1880 mask to zero.
1882 Parameters
1883 ----------
1884 exposure : `lsst.afw.image.Exposure`
1885 The raw exposure to be converted.
1887 Returns
1888 -------
1889 newexposure : `lsst.afw.image.Exposure`
1890 The input ``exposure``, converted to floating point pixels.
1892 Raises
1893 ------
1894 RuntimeError
1895 Raised if the exposure type cannot be converted to float.
1897 """
1898 if isinstance(exposure, afwImage.ExposureF):
1899 # Nothing to be done
1900 self.log.debug("Exposure already of type float.")
1901 return exposure
1902 if not hasattr(exposure, "convertF"):
1903 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1905 newexposure = exposure.convertF()
1906 newexposure.variance[:] = 1
1907 newexposure.mask[:] = 0x0
1909 return newexposure
1911 def maskAmplifier(self, ccdExposure, amp, defects):
1912 """Identify bad amplifiers, saturated and suspect pixels.
1914 Parameters
1915 ----------
1916 ccdExposure : `lsst.afw.image.Exposure`
1917 Input exposure to be masked.
1918 amp : `lsst.afw.table.AmpInfoCatalog`
1919 Catalog of parameters defining the amplifier on this
1920 exposure to mask.
1921 defects : `lsst.ip.isr.Defects`
1922 List of defects. Used to determine if the entire
1923 amplifier is bad.
1925 Returns
1926 -------
1927 badAmp : `Bool`
1928 If this is true, the entire amplifier area is covered by
1929 defects and unusable.
1931 """
1932 maskedImage = ccdExposure.getMaskedImage()
1934 badAmp = False
1936 # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1937 # comparison with current defects definition.
1938 if defects is not None:
1939 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1941 # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1942 # association with pixels in current ccdExposure).
1943 if badAmp:
1944 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1945 afwImage.PARENT)
1946 maskView = dataView.getMask()
1947 maskView |= maskView.getPlaneBitMask("BAD")
1948 del maskView
1949 return badAmp
1951 # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1952 # Saturation and suspect pixels can be masked now, though.
1953 limits = dict()
1954 if self.config.doSaturation and not badAmp:
1955 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1956 if self.config.doSuspect and not badAmp:
1957 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1958 if math.isfinite(self.config.saturation):
1959 limits.update({self.config.saturatedMaskName: self.config.saturation})
1961 for maskName, maskThreshold in limits.items():
1962 if not math.isnan(maskThreshold):
1963 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1964 isrFunctions.makeThresholdMask(
1965 maskedImage=dataView,
1966 threshold=maskThreshold,
1967 growFootprints=0,
1968 maskName=maskName
1969 )
1971 # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1972 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1973 afwImage.PARENT)
1974 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1975 self.config.suspectMaskName])
1976 if numpy.all(maskView.getArray() & maskVal > 0):
1977 badAmp = True
1978 maskView |= maskView.getPlaneBitMask("BAD")
1980 return badAmp
1982 def overscanCorrection(self, ccdExposure, amp):
1983 """Apply overscan correction in place.
1985 This method does initial pixel rejection of the overscan
1986 region. The overscan can also be optionally segmented to
1987 allow for discontinuous overscan responses to be fit
1988 separately. The actual overscan subtraction is performed by
1989 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1990 which is called here after the amplifier is preprocessed.
1992 Parameters
1993 ----------
1994 ccdExposure : `lsst.afw.image.Exposure`
1995 Exposure to have overscan correction performed.
1996 amp : `lsst.afw.cameraGeom.Amplifer`
1997 The amplifier to consider while correcting the overscan.
1999 Returns
2000 -------
2001 overscanResults : `lsst.pipe.base.Struct`
2002 Result struct with components:
2003 - ``imageFit`` : scalar or `lsst.afw.image.Image`
2004 Value or fit subtracted from the amplifier image data.
2005 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
2006 Value or fit subtracted from the overscan image data.
2007 - ``overscanImage`` : `lsst.afw.image.Image`
2008 Image of the overscan region with the overscan
2009 correction applied. This quantity is used to estimate
2010 the amplifier read noise empirically.
2012 Raises
2013 ------
2014 RuntimeError
2015 Raised if the ``amp`` does not contain raw pixel information.
2017 See Also
2018 --------
2019 lsst.ip.isr.isrFunctions.overscanCorrection
2020 """
2021 if amp.getRawHorizontalOverscanBBox().isEmpty():
2022 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
2023 return None
2025 statControl = afwMath.StatisticsControl()
2026 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2028 # Determine the bounding boxes
2029 dataBBox = amp.getRawDataBBox()
2030 oscanBBox = amp.getRawHorizontalOverscanBBox()
2031 dx0 = 0
2032 dx1 = 0
2034 prescanBBox = amp.getRawPrescanBBox()
2035 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
2036 dx0 += self.config.overscanNumLeadingColumnsToSkip
2037 dx1 -= self.config.overscanNumTrailingColumnsToSkip
2038 else:
2039 dx0 += self.config.overscanNumTrailingColumnsToSkip
2040 dx1 -= self.config.overscanNumLeadingColumnsToSkip
2042 # Determine if we need to work on subregions of the amplifier and overscan.
2043 imageBBoxes = []
2044 overscanBBoxes = []
2046 if ((self.config.overscanBiasJump
2047 and self.config.overscanBiasJumpLocation)
2048 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2049 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
2050 self.config.overscanBiasJumpDevices)):
2051 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
2052 yLower = self.config.overscanBiasJumpLocation
2053 yUpper = dataBBox.getHeight() - yLower
2054 else:
2055 yUpper = self.config.overscanBiasJumpLocation
2056 yLower = dataBBox.getHeight() - yUpper
2058 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2059 lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
2060 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2061 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2062 yLower)))
2064 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
2065 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
2066 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
2067 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2068 yUpper)))
2069 else:
2070 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2071 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
2072 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2073 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2074 oscanBBox.getHeight())))
2076 # Perform overscan correction on subregions, ensuring saturated pixels are masked.
2077 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
2078 ampImage = ccdExposure.maskedImage[imageBBox]
2079 overscanImage = ccdExposure.maskedImage[overscanBBox]
2081 overscanArray = overscanImage.image.array
2082 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2083 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2084 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
2086 statControl = afwMath.StatisticsControl()
2087 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2089 overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp)
2091 # Measure average overscan levels and record them in the metadata.
2092 levelStat = afwMath.MEDIAN
2093 sigmaStat = afwMath.STDEVCLIP
2095 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2096 self.config.qa.flatness.nIter)
2097 metadata = ccdExposure.getMetadata()
2098 ampNum = amp.getName()
2099 # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
2100 if isinstance(overscanResults.overscanFit, float):
2101 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2102 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2103 else:
2104 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2105 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2106 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2108 return overscanResults
2110 def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None):
2111 """Set the variance plane using the gain and read noise
2113 The read noise is calculated from the ``overscanImage`` if the
2114 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2115 the value from the amplifier data is used.
2117 Parameters
2118 ----------
2119 ampExposure : `lsst.afw.image.Exposure`
2120 Exposure to process.
2121 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2122 Amplifier detector data.
2123 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2124 Image of overscan, required only for empirical read noise.
2125 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
2126 PTC dataset containing the gains and read noise.
2129 Raises
2130 ------
2131 RuntimeError
2132 Raised if either ``usePtcGains`` of ``usePtcReadNoise``
2133 are ``True``, but ptcDataset is not provided.
2135 Raised if ```doEmpiricalReadNoise`` is ``True`` but
2136 ``overscanImage`` is ``None``.
2138 See also
2139 --------
2140 lsst.ip.isr.isrFunctions.updateVariance
2141 """
2142 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2143 if self.config.usePtcGains:
2144 if ptcDataset is None:
2145 raise RuntimeError("No ptcDataset provided to use PTC gains.")
2146 else:
2147 gain = ptcDataset.gain[amp.getName()]
2148 self.log.info("Using gain from Photon Transfer Curve.")
2149 else:
2150 gain = amp.getGain()
2152 if math.isnan(gain):
2153 gain = 1.0
2154 self.log.warn("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2155 elif gain <= 0:
2156 patchedGain = 1.0
2157 self.log.warn("Gain for amp %s == %g <= 0; setting to %f.",
2158 amp.getName(), gain, patchedGain)
2159 gain = patchedGain
2161 if self.config.doEmpiricalReadNoise and overscanImage is None:
2162 raise RuntimeError("Overscan is none for EmpiricalReadNoise.")
2164 if self.config.doEmpiricalReadNoise and overscanImage is not None:
2165 stats = afwMath.StatisticsControl()
2166 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2167 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2168 self.log.info("Calculated empirical read noise for amp %s: %f.",
2169 amp.getName(), readNoise)
2170 elif self.config.usePtcReadNoise:
2171 if ptcDataset is None:
2172 raise RuntimeError("No ptcDataset provided to use PTC readnoise.")
2173 else:
2174 readNoise = ptcDataset.noise[amp.getName()]
2175 self.log.info("Using read noise from Photon Transfer Curve.")
2176 else:
2177 readNoise = amp.getReadNoise()
2179 isrFunctions.updateVariance(
2180 maskedImage=ampExposure.getMaskedImage(),
2181 gain=gain,
2182 readNoise=readNoise,
2183 )
2185 def darkCorrection(self, exposure, darkExposure, invert=False):
2186 """Apply dark correction in place.
2188 Parameters
2189 ----------
2190 exposure : `lsst.afw.image.Exposure`
2191 Exposure to process.
2192 darkExposure : `lsst.afw.image.Exposure`
2193 Dark exposure of the same size as ``exposure``.
2194 invert : `Bool`, optional
2195 If True, re-add the dark to an already corrected image.
2197 Raises
2198 ------
2199 RuntimeError
2200 Raised if either ``exposure`` or ``darkExposure`` do not
2201 have their dark time defined.
2203 See Also
2204 --------
2205 lsst.ip.isr.isrFunctions.darkCorrection
2206 """
2207 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2208 if math.isnan(expScale):
2209 raise RuntimeError("Exposure darktime is NAN.")
2210 if darkExposure.getInfo().getVisitInfo() is not None \
2211 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2212 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2213 else:
2214 # DM-17444: darkExposure.getInfo.getVisitInfo() is None
2215 # so getDarkTime() does not exist.
2216 self.log.warn("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2217 darkScale = 1.0
2219 isrFunctions.darkCorrection(
2220 maskedImage=exposure.getMaskedImage(),
2221 darkMaskedImage=darkExposure.getMaskedImage(),
2222 expScale=expScale,
2223 darkScale=darkScale,
2224 invert=invert,
2225 trimToFit=self.config.doTrimToMatchCalib
2226 )
2228 def doLinearize(self, detector):
2229 """Check if linearization is needed for the detector cameraGeom.
2231 Checks config.doLinearize and the linearity type of the first
2232 amplifier.
2234 Parameters
2235 ----------
2236 detector : `lsst.afw.cameraGeom.Detector`
2237 Detector to get linearity type from.
2239 Returns
2240 -------
2241 doLinearize : `Bool`
2242 If True, linearization should be performed.
2243 """
2244 return self.config.doLinearize and \
2245 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2247 def flatCorrection(self, exposure, flatExposure, invert=False):
2248 """Apply flat correction in place.
2250 Parameters
2251 ----------
2252 exposure : `lsst.afw.image.Exposure`
2253 Exposure to process.
2254 flatExposure : `lsst.afw.image.Exposure`
2255 Flat exposure of the same size as ``exposure``.
2256 invert : `Bool`, optional
2257 If True, unflatten an already flattened image.
2259 See Also
2260 --------
2261 lsst.ip.isr.isrFunctions.flatCorrection
2262 """
2263 isrFunctions.flatCorrection(
2264 maskedImage=exposure.getMaskedImage(),
2265 flatMaskedImage=flatExposure.getMaskedImage(),
2266 scalingType=self.config.flatScalingType,
2267 userScale=self.config.flatUserScale,
2268 invert=invert,
2269 trimToFit=self.config.doTrimToMatchCalib
2270 )
2272 def saturationDetection(self, exposure, amp):
2273 """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2275 Parameters
2276 ----------
2277 exposure : `lsst.afw.image.Exposure`
2278 Exposure to process. Only the amplifier DataSec is processed.
2279 amp : `lsst.afw.table.AmpInfoCatalog`
2280 Amplifier detector data.
2282 See Also
2283 --------
2284 lsst.ip.isr.isrFunctions.makeThresholdMask
2285 """
2286 if not math.isnan(amp.getSaturation()):
2287 maskedImage = exposure.getMaskedImage()
2288 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2289 isrFunctions.makeThresholdMask(
2290 maskedImage=dataView,
2291 threshold=amp.getSaturation(),
2292 growFootprints=0,
2293 maskName=self.config.saturatedMaskName,
2294 )
2296 def saturationInterpolation(self, exposure):
2297 """Interpolate over saturated pixels, in place.
2299 This method should be called after `saturationDetection`, to
2300 ensure that the saturated pixels have been identified in the
2301 SAT mask. It should also be called after `assembleCcd`, since
2302 saturated regions may cross amplifier boundaries.
2304 Parameters
2305 ----------
2306 exposure : `lsst.afw.image.Exposure`
2307 Exposure to process.
2309 See Also
2310 --------
2311 lsst.ip.isr.isrTask.saturationDetection
2312 lsst.ip.isr.isrFunctions.interpolateFromMask
2313 """
2314 isrFunctions.interpolateFromMask(
2315 maskedImage=exposure.getMaskedImage(),
2316 fwhm=self.config.fwhm,
2317 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2318 maskNameList=list(self.config.saturatedMaskName),
2319 )
2321 def suspectDetection(self, exposure, amp):
2322 """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2324 Parameters
2325 ----------
2326 exposure : `lsst.afw.image.Exposure`
2327 Exposure to process. Only the amplifier DataSec is processed.
2328 amp : `lsst.afw.table.AmpInfoCatalog`
2329 Amplifier detector data.
2331 See Also
2332 --------
2333 lsst.ip.isr.isrFunctions.makeThresholdMask
2335 Notes
2336 -----
2337 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2338 This is intended to indicate pixels that may be affected by unknown systematics;
2339 for example if non-linearity corrections above a certain level are unstable
2340 then that would be a useful value for suspectLevel. A value of `nan` indicates
2341 that no such level exists and no pixels are to be masked as suspicious.
2342 """
2343 suspectLevel = amp.getSuspectLevel()
2344 if math.isnan(suspectLevel):
2345 return
2347 maskedImage = exposure.getMaskedImage()
2348 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2349 isrFunctions.makeThresholdMask(
2350 maskedImage=dataView,
2351 threshold=suspectLevel,
2352 growFootprints=0,
2353 maskName=self.config.suspectMaskName,
2354 )
2356 def maskDefect(self, exposure, defectBaseList):
2357 """Mask defects using mask plane "BAD", in place.
2359 Parameters
2360 ----------
2361 exposure : `lsst.afw.image.Exposure`
2362 Exposure to process.
2363 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2364 `lsst.afw.image.DefectBase`.
2365 List of defects to mask.
2367 Notes
2368 -----
2369 Call this after CCD assembly, since defects may cross amplifier boundaries.
2370 """
2371 maskedImage = exposure.getMaskedImage()
2372 if not isinstance(defectBaseList, Defects):
2373 # Promotes DefectBase to Defect
2374 defectList = Defects(defectBaseList)
2375 else:
2376 defectList = defectBaseList
2377 defectList.maskPixels(maskedImage, maskName="BAD")
2379 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2380 """Mask edge pixels with applicable mask plane.
2382 Parameters
2383 ----------
2384 exposure : `lsst.afw.image.Exposure`
2385 Exposure to process.
2386 numEdgePixels : `int`, optional
2387 Number of edge pixels to mask.
2388 maskPlane : `str`, optional
2389 Mask plane name to use.
2390 level : `str`, optional
2391 Level at which to mask edges.
2392 """
2393 maskedImage = exposure.getMaskedImage()
2394 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2396 if numEdgePixels > 0:
2397 if level == 'DETECTOR':
2398 boxes = [maskedImage.getBBox()]
2399 elif level == 'AMP':
2400 boxes = [amp.getBBox() for amp in exposure.getDetector()]
2402 for box in boxes:
2403 # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2404 subImage = maskedImage[box]
2405 box.grow(-numEdgePixels)
2406 # Mask pixels outside box
2407 SourceDetectionTask.setEdgeBits(
2408 subImage,
2409 box,
2410 maskBitMask)
2412 def maskAndInterpolateDefects(self, exposure, defectBaseList):
2413 """Mask and interpolate defects using mask plane "BAD", in place.
2415 Parameters
2416 ----------
2417 exposure : `lsst.afw.image.Exposure`
2418 Exposure to process.
2419 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2420 `lsst.afw.image.DefectBase`.
2421 List of defects to mask and interpolate.
2423 See Also
2424 --------
2425 lsst.ip.isr.isrTask.maskDefect
2426 """
2427 self.maskDefect(exposure, defectBaseList)
2428 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2429 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2430 isrFunctions.interpolateFromMask(
2431 maskedImage=exposure.getMaskedImage(),
2432 fwhm=self.config.fwhm,
2433 growSaturatedFootprints=0,
2434 maskNameList=["BAD"],
2435 )
2437 def maskNan(self, exposure):
2438 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2440 Parameters
2441 ----------
2442 exposure : `lsst.afw.image.Exposure`
2443 Exposure to process.
2445 Notes
2446 -----
2447 We mask over all non-finite values (NaN, inf), including those
2448 that are masked with other bits (because those may or may not be
2449 interpolated over later, and we want to remove all NaN/infs).
2450 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2451 preserve the historical name.
2452 """
2453 maskedImage = exposure.getMaskedImage()
2455 # Find and mask NaNs
2456 maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2457 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2458 numNans = maskNans(maskedImage, maskVal)
2459 self.metadata.set("NUMNANS", numNans)
2460 if numNans > 0:
2461 self.log.warn("There were %d unmasked NaNs.", numNans)
2463 def maskAndInterpolateNan(self, exposure):
2464 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2465 in place.
2467 Parameters
2468 ----------
2469 exposure : `lsst.afw.image.Exposure`
2470 Exposure to process.
2472 See Also
2473 --------
2474 lsst.ip.isr.isrTask.maskNan
2475 """
2476 self.maskNan(exposure)
2477 isrFunctions.interpolateFromMask(
2478 maskedImage=exposure.getMaskedImage(),
2479 fwhm=self.config.fwhm,
2480 growSaturatedFootprints=0,
2481 maskNameList=["UNMASKEDNAN"],
2482 )
2484 def measureBackground(self, exposure, IsrQaConfig=None):
2485 """Measure the image background in subgrids, for quality control purposes.
2487 Parameters
2488 ----------
2489 exposure : `lsst.afw.image.Exposure`
2490 Exposure to process.
2491 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2492 Configuration object containing parameters on which background
2493 statistics and subgrids to use.
2494 """
2495 if IsrQaConfig is not None:
2496 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2497 IsrQaConfig.flatness.nIter)
2498 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2499 statsControl.setAndMask(maskVal)
2500 maskedImage = exposure.getMaskedImage()
2501 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2502 skyLevel = stats.getValue(afwMath.MEDIAN)
2503 skySigma = stats.getValue(afwMath.STDEVCLIP)
2504 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2505 metadata = exposure.getMetadata()
2506 metadata.set('SKYLEVEL', skyLevel)
2507 metadata.set('SKYSIGMA', skySigma)
2509 # calcluating flatlevel over the subgrids
2510 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2511 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2512 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2513 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2514 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2515 skyLevels = numpy.zeros((nX, nY))
2517 for j in range(nY):
2518 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2519 for i in range(nX):
2520 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2522 xLLC = xc - meshXHalf
2523 yLLC = yc - meshYHalf
2524 xURC = xc + meshXHalf - 1
2525 yURC = yc + meshYHalf - 1
2527 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2528 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2530 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2532 good = numpy.where(numpy.isfinite(skyLevels))
2533 skyMedian = numpy.median(skyLevels[good])
2534 flatness = (skyLevels[good] - skyMedian) / skyMedian
2535 flatness_rms = numpy.std(flatness)
2536 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2538 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2539 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2540 nX, nY, flatness_pp, flatness_rms)
2542 metadata.set('FLATNESS_PP', float(flatness_pp))
2543 metadata.set('FLATNESS_RMS', float(flatness_rms))
2544 metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2545 metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2546 metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2548 def roughZeroPoint(self, exposure):
2549 """Set an approximate magnitude zero point for the exposure.
2551 Parameters
2552 ----------
2553 exposure : `lsst.afw.image.Exposure`
2554 Exposure to process.
2555 """
2556 filterLabel = exposure.getFilterLabel()
2557 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2559 if physicalFilter in self.config.fluxMag0T1:
2560 fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2561 else:
2562 self.log.warn("No rough magnitude zero point defined for filter {}.".format(physicalFilter))
2563 fluxMag0 = self.config.defaultFluxMag0T1
2565 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2566 if not expTime > 0: # handle NaN as well as <= 0
2567 self.log.warn("Non-positive exposure time; skipping rough zero point.")
2568 return
2570 self.log.info("Setting rough magnitude zero point for filter {}: {}".
2571 format(physicalFilter, 2.5*math.log10(fluxMag0*expTime)))
2572 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2574 def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2575 """Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2577 Parameters
2578 ----------
2579 ccdExposure : `lsst.afw.image.Exposure`
2580 Exposure to process.
2581 fpPolygon : `lsst.afw.geom.Polygon`
2582 Polygon in focal plane coordinates.
2583 """
2584 # Get ccd corners in focal plane coordinates
2585 ccd = ccdExposure.getDetector()
2586 fpCorners = ccd.getCorners(FOCAL_PLANE)
2587 ccdPolygon = Polygon(fpCorners)
2589 # Get intersection of ccd corners with fpPolygon
2590 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2592 # Transform back to pixel positions and build new polygon
2593 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2594 validPolygon = Polygon(ccdPoints)
2595 ccdExposure.getInfo().setValidPolygon(validPolygon)
2597 @contextmanager
2598 def flatContext(self, exp, flat, dark=None):
2599 """Context manager that applies and removes flats and darks,
2600 if the task is configured to apply them.
2602 Parameters
2603 ----------
2604 exp : `lsst.afw.image.Exposure`
2605 Exposure to process.
2606 flat : `lsst.afw.image.Exposure`
2607 Flat exposure the same size as ``exp``.
2608 dark : `lsst.afw.image.Exposure`, optional
2609 Dark exposure the same size as ``exp``.
2611 Yields
2612 ------
2613 exp : `lsst.afw.image.Exposure`
2614 The flat and dark corrected exposure.
2615 """
2616 if self.config.doDark and dark is not None:
2617 self.darkCorrection(exp, dark)
2618 if self.config.doFlat:
2619 self.flatCorrection(exp, flat)
2620 try:
2621 yield exp
2622 finally:
2623 if self.config.doFlat:
2624 self.flatCorrection(exp, flat, invert=True)
2625 if self.config.doDark and dark is not None:
2626 self.darkCorrection(exp, dark, invert=True)
2628 def debugView(self, exposure, stepname):
2629 """Utility function to examine ISR exposure at different stages.
2631 Parameters
2632 ----------
2633 exposure : `lsst.afw.image.Exposure`
2634 Exposure to view.
2635 stepname : `str`
2636 State of processing to view.
2637 """
2638 frame = getDebugFrame(self._display, stepname)
2639 if frame:
2640 display = getDisplay(frame)
2641 display.scale('asinh', 'zscale')
2642 display.mtv(exposure)
2643 prompt = "Press Enter to continue [c]... "
2644 while True:
2645 ans = input(prompt).lower()
2646 if ans in ("", "c",):
2647 break
2650class FakeAmp(object):
2651 """A Detector-like object that supports returning gain and saturation level
2653 This is used when the input exposure does not have a detector.
2655 Parameters
2656 ----------
2657 exposure : `lsst.afw.image.Exposure`
2658 Exposure to generate a fake amplifier for.
2659 config : `lsst.ip.isr.isrTaskConfig`
2660 Configuration to apply to the fake amplifier.
2661 """
2663 def __init__(self, exposure, config):
2664 self._bbox = exposure.getBBox(afwImage.LOCAL)
2665 self._RawHorizontalOverscanBBox = lsst.geom.Box2I()
2666 self._gain = config.gain
2667 self._readNoise = config.readNoise
2668 self._saturation = config.saturation
2670 def getBBox(self):
2671 return self._bbox
2673 def getRawBBox(self):
2674 return self._bbox
2676 def getRawHorizontalOverscanBBox(self):
2677 return self._RawHorizontalOverscanBBox
2679 def getGain(self):
2680 return self._gain
2682 def getReadNoise(self):
2683 return self._readNoise
2685 def getSaturation(self):
2686 return self._saturation
2688 def getSuspectLevel(self):
2689 return float("NaN")
2692class RunIsrConfig(pexConfig.Config):
2693 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2696class RunIsrTask(pipeBase.CmdLineTask):
2697 """Task to wrap the default IsrTask to allow it to be retargeted.
2699 The standard IsrTask can be called directly from a command line
2700 program, but doing so removes the ability of the task to be
2701 retargeted. As most cameras override some set of the IsrTask
2702 methods, this would remove those data-specific methods in the
2703 output post-ISR images. This wrapping class fixes the issue,
2704 allowing identical post-ISR images to be generated by both the
2705 processCcd and isrTask code.
2706 """
2707 ConfigClass = RunIsrConfig
2708 _DefaultName = "runIsr"
2710 def __init__(self, *args, **kwargs):
2711 super().__init__(*args, **kwargs)
2712 self.makeSubtask("isr")
2714 def runDataRef(self, dataRef):
2715 """
2716 Parameters
2717 ----------
2718 dataRef : `lsst.daf.persistence.ButlerDataRef`
2719 data reference of the detector data to be processed
2721 Returns
2722 -------
2723 result : `pipeBase.Struct`
2724 Result struct with component:
2726 - exposure : `lsst.afw.image.Exposure`
2727 Post-ISR processed exposure.
2728 """
2729 return self.isr.runDataRef(dataRef)