Coverage for python/lsst/ip/isr/isrTask.py : 16%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of ip_isr.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22import math
23import numpy
25import lsst.geom
26import lsst.afw.image as afwImage
27import lsst.afw.math as afwMath
28import lsst.pex.config as pexConfig
29import lsst.pipe.base as pipeBase
30import lsst.pipe.base.connectionTypes as cT
32from contextlib import contextmanager
33from lsstDebug import getDebugFrame
35from lsst.afw.cameraGeom import (PIXELS, FOCAL_PLANE, NullLinearityType,
36 ReadoutCorner)
37from lsst.afw.display import getDisplay
38from lsst.afw.geom import Polygon
39from lsst.daf.persistence import ButlerDataRef
40from lsst.daf.persistence.butler import NoResults
41from lsst.meas.algorithms.detection import SourceDetectionTask
43from . import isrFunctions
44from . import isrQa
45from . import linearize
46from .defects import Defects
48from .assembleCcdTask import AssembleCcdTask
49from .crosstalk import CrosstalkTask, CrosstalkCalib
50from .fringe import FringeTask
51from .isr import maskNans
52from .masking import MaskingTask
53from .overscan import OverscanCorrectionTask
54from .straylight import StrayLightTask
55from .vignette import VignetteTask
56from lsst.daf.butler import DimensionGraph
59__all__ = ["IsrTask", "IsrTaskConfig", "RunIsrTask", "RunIsrConfig"]
62def crosstalkSourceLookup(datasetType, registry, quantumDataId, collections):
63 """Lookup function to identify crosstalkSource entries.
65 This should return an empty list under most circumstances. Only
66 when inter-chip crosstalk has been identified should this be
67 populated.
69 This will be unused until DM-25348 resolves the quantum graph
70 generation issue.
72 Parameters
73 ----------
74 datasetType : `str`
75 Dataset to lookup.
76 registry : `lsst.daf.butler.Registry`
77 Butler registry to query.
78 quantumDataId : `lsst.daf.butler.ExpandedDataCoordinate`
79 Data id to transform to identify crosstalkSources. The
80 ``detector`` entry will be stripped.
81 collections : `lsst.daf.butler.CollectionSearch`
82 Collections to search through.
84 Returns
85 -------
86 results : `list` [`lsst.daf.butler.DatasetRef`]
87 List of datasets that match the query that will be used as
88 crosstalkSources.
89 """
90 newDataId = quantumDataId.subset(DimensionGraph(registry.dimensions, names=["instrument", "exposure"]))
91 results = list(registry.queryDatasets(datasetType,
92 collections=collections,
93 dataId=newDataId,
94 findFirst=True,
95 ).expanded())
96 return results
99class IsrTaskConnections(pipeBase.PipelineTaskConnections,
100 dimensions={"instrument", "exposure", "detector"},
101 defaultTemplates={}):
102 ccdExposure = cT.Input(
103 name="raw",
104 doc="Input exposure to process.",
105 storageClass="Exposure",
106 dimensions=["instrument", "exposure", "detector"],
107 )
108 camera = cT.PrerequisiteInput(
109 name="camera",
110 storageClass="Camera",
111 doc="Input camera to construct complete exposures.",
112 dimensions=["instrument"],
113 isCalibration=True,
114 )
116 crosstalk = cT.PrerequisiteInput(
117 name="crosstalk",
118 doc="Input crosstalk object",
119 storageClass="CrosstalkCalib",
120 dimensions=["instrument", "detector"],
121 isCalibration=True,
122 minimum=0, # can fall back to cameraGeom
123 )
124 # TODO: DM-25348. This does not work yet to correctly load
125 # possible crosstalk sources.
126 crosstalkSources = cT.PrerequisiteInput(
127 name="isrOverscanCorrected",
128 doc="Overscan corrected input images.",
129 storageClass="Exposure",
130 dimensions=["instrument", "exposure", "detector"],
131 deferLoad=True,
132 multiple=True,
133 lookupFunction=crosstalkSourceLookup,
134 minimum=0, # not needed for all instruments, no config to control this
135 )
136 bias = cT.PrerequisiteInput(
137 name="bias",
138 doc="Input bias calibration.",
139 storageClass="ExposureF",
140 dimensions=["instrument", "detector"],
141 isCalibration=True,
142 )
143 dark = cT.PrerequisiteInput(
144 name='dark',
145 doc="Input dark calibration.",
146 storageClass="ExposureF",
147 dimensions=["instrument", "detector"],
148 isCalibration=True,
149 )
150 flat = cT.PrerequisiteInput(
151 name="flat",
152 doc="Input flat calibration.",
153 storageClass="ExposureF",
154 dimensions=["instrument", "physical_filter", "detector"],
155 isCalibration=True,
156 )
157 ptc = cT.PrerequisiteInput(
158 name="ptc",
159 doc="Input Photon Transfer Curve dataset",
160 storageClass="PhotonTransferCurveDataset",
161 dimensions=["instrument", "detector"],
162 isCalibration=True,
163 )
164 fringes = cT.PrerequisiteInput(
165 name="fringe",
166 doc="Input fringe calibration.",
167 storageClass="ExposureF",
168 dimensions=["instrument", "physical_filter", "detector"],
169 isCalibration=True,
170 minimum=0, # only needed for some bands, even when enabled
171 )
172 strayLightData = cT.PrerequisiteInput(
173 name='yBackground',
174 doc="Input stray light calibration.",
175 storageClass="StrayLightData",
176 dimensions=["instrument", "physical_filter", "detector"],
177 isCalibration=True,
178 minimum=0, # only needed for some bands, even when enabled
179 )
180 bfKernel = cT.PrerequisiteInput(
181 name='bfKernel',
182 doc="Input brighter-fatter kernel.",
183 storageClass="NumpyArray",
184 dimensions=["instrument"],
185 isCalibration=True,
186 minimum=0, # can use either bfKernel or newBFKernel
187 )
188 newBFKernel = cT.PrerequisiteInput(
189 name='brighterFatterKernel',
190 doc="Newer complete kernel + gain solutions.",
191 storageClass="BrighterFatterKernel",
192 dimensions=["instrument", "detector"],
193 isCalibration=True,
194 minimum=0, # can use either bfKernel or newBFKernel
195 )
196 defects = cT.PrerequisiteInput(
197 name='defects',
198 doc="Input defect tables.",
199 storageClass="Defects",
200 dimensions=["instrument", "detector"],
201 isCalibration=True,
202 )
203 linearizer = cT.PrerequisiteInput(
204 name='linearizer',
205 storageClass="Linearizer",
206 doc="Linearity correction calibration.",
207 dimensions=["instrument", "detector"],
208 isCalibration=True,
209 minimum=0, # can fall back to cameraGeom
210 )
211 opticsTransmission = cT.PrerequisiteInput(
212 name="transmission_optics",
213 storageClass="TransmissionCurve",
214 doc="Transmission curve due to the optics.",
215 dimensions=["instrument"],
216 isCalibration=True,
217 )
218 filterTransmission = cT.PrerequisiteInput(
219 name="transmission_filter",
220 storageClass="TransmissionCurve",
221 doc="Transmission curve due to the filter.",
222 dimensions=["instrument", "physical_filter"],
223 isCalibration=True,
224 )
225 sensorTransmission = cT.PrerequisiteInput(
226 name="transmission_sensor",
227 storageClass="TransmissionCurve",
228 doc="Transmission curve due to the sensor.",
229 dimensions=["instrument", "detector"],
230 isCalibration=True,
231 )
232 atmosphereTransmission = cT.PrerequisiteInput(
233 name="transmission_atmosphere",
234 storageClass="TransmissionCurve",
235 doc="Transmission curve due to the atmosphere.",
236 dimensions=["instrument"],
237 isCalibration=True,
238 )
239 illumMaskedImage = cT.PrerequisiteInput(
240 name="illum",
241 doc="Input illumination correction.",
242 storageClass="MaskedImageF",
243 dimensions=["instrument", "physical_filter", "detector"],
244 isCalibration=True,
245 )
247 outputExposure = cT.Output(
248 name='postISRCCD',
249 doc="Output ISR processed exposure.",
250 storageClass="Exposure",
251 dimensions=["instrument", "exposure", "detector"],
252 )
253 preInterpExposure = cT.Output(
254 name='preInterpISRCCD',
255 doc="Output ISR processed exposure, with pixels left uninterpolated.",
256 storageClass="ExposureF",
257 dimensions=["instrument", "exposure", "detector"],
258 )
259 outputOssThumbnail = cT.Output(
260 name="OssThumb",
261 doc="Output Overscan-subtracted thumbnail image.",
262 storageClass="Thumbnail",
263 dimensions=["instrument", "exposure", "detector"],
264 )
265 outputFlattenedThumbnail = cT.Output(
266 name="FlattenedThumb",
267 doc="Output flat-corrected thumbnail image.",
268 storageClass="Thumbnail",
269 dimensions=["instrument", "exposure", "detector"],
270 )
272 def __init__(self, *, config=None):
273 super().__init__(config=config)
275 if config.doBias is not True:
276 self.prerequisiteInputs.discard("bias")
277 if config.doLinearize is not True:
278 self.prerequisiteInputs.discard("linearizer")
279 if config.doCrosstalk is not True:
280 self.inputs.discard("crosstalkSources")
281 self.prerequisiteInputs.discard("crosstalk")
282 if config.doBrighterFatter is not True:
283 self.prerequisiteInputs.discard("bfKernel")
284 self.prerequisiteInputs.discard("newBFKernel")
285 if config.doDefect is not True:
286 self.prerequisiteInputs.discard("defects")
287 if config.doDark is not True:
288 self.prerequisiteInputs.discard("dark")
289 if config.doFlat is not True:
290 self.prerequisiteInputs.discard("flat")
291 if config.doFringe is not True:
292 self.prerequisiteInputs.discard("fringe")
293 if config.doStrayLight is not True:
294 self.prerequisiteInputs.discard("strayLightData")
295 if config.usePtcGains is not True and config.usePtcReadNoise is not True:
296 self.prerequisiteInputs.discard("ptc")
297 if config.doAttachTransmissionCurve is not True:
298 self.prerequisiteInputs.discard("opticsTransmission")
299 self.prerequisiteInputs.discard("filterTransmission")
300 self.prerequisiteInputs.discard("sensorTransmission")
301 self.prerequisiteInputs.discard("atmosphereTransmission")
302 if config.doUseOpticsTransmission is not True:
303 self.prerequisiteInputs.discard("opticsTransmission")
304 if config.doUseFilterTransmission is not True:
305 self.prerequisiteInputs.discard("filterTransmission")
306 if config.doUseSensorTransmission is not True:
307 self.prerequisiteInputs.discard("sensorTransmission")
308 if config.doUseAtmosphereTransmission is not True:
309 self.prerequisiteInputs.discard("atmosphereTransmission")
310 if config.doIlluminationCorrection is not True:
311 self.prerequisiteInputs.discard("illumMaskedImage")
313 if config.doWrite is not True:
314 self.outputs.discard("outputExposure")
315 self.outputs.discard("preInterpExposure")
316 self.outputs.discard("outputFlattenedThumbnail")
317 self.outputs.discard("outputOssThumbnail")
318 if config.doSaveInterpPixels is not True:
319 self.outputs.discard("preInterpExposure")
320 if config.qa.doThumbnailOss is not True:
321 self.outputs.discard("outputOssThumbnail")
322 if config.qa.doThumbnailFlattened is not True:
323 self.outputs.discard("outputFlattenedThumbnail")
326class IsrTaskConfig(pipeBase.PipelineTaskConfig,
327 pipelineConnections=IsrTaskConnections):
328 """Configuration parameters for IsrTask.
330 Items are grouped in the order in which they are executed by the task.
331 """
332 datasetType = pexConfig.Field(
333 dtype=str,
334 doc="Dataset type for input data; users will typically leave this alone, "
335 "but camera-specific ISR tasks will override it",
336 default="raw",
337 )
339 fallbackFilterName = pexConfig.Field(
340 dtype=str,
341 doc="Fallback default filter name for calibrations.",
342 optional=True
343 )
344 useFallbackDate = pexConfig.Field(
345 dtype=bool,
346 doc="Pass observation date when using fallback filter.",
347 default=False,
348 )
349 expectWcs = pexConfig.Field(
350 dtype=bool,
351 default=True,
352 doc="Expect input science images to have a WCS (set False for e.g. spectrographs)."
353 )
354 fwhm = pexConfig.Field(
355 dtype=float,
356 doc="FWHM of PSF in arcseconds.",
357 default=1.0,
358 )
359 qa = pexConfig.ConfigField(
360 dtype=isrQa.IsrQaConfig,
361 doc="QA related configuration options.",
362 )
364 # Image conversion configuration
365 doConvertIntToFloat = pexConfig.Field(
366 dtype=bool,
367 doc="Convert integer raw images to floating point values?",
368 default=True,
369 )
371 # Saturated pixel handling.
372 doSaturation = pexConfig.Field(
373 dtype=bool,
374 doc="Mask saturated pixels? NB: this is totally independent of the"
375 " interpolation option - this is ONLY setting the bits in the mask."
376 " To have them interpolated make sure doSaturationInterpolation=True",
377 default=True,
378 )
379 saturatedMaskName = pexConfig.Field(
380 dtype=str,
381 doc="Name of mask plane to use in saturation detection and interpolation",
382 default="SAT",
383 )
384 saturation = pexConfig.Field(
385 dtype=float,
386 doc="The saturation level to use if no Detector is present in the Exposure (ignored if NaN)",
387 default=float("NaN"),
388 )
389 growSaturationFootprintSize = pexConfig.Field(
390 dtype=int,
391 doc="Number of pixels by which to grow the saturation footprints",
392 default=1,
393 )
395 # Suspect pixel handling.
396 doSuspect = pexConfig.Field(
397 dtype=bool,
398 doc="Mask suspect pixels?",
399 default=False,
400 )
401 suspectMaskName = pexConfig.Field(
402 dtype=str,
403 doc="Name of mask plane to use for suspect pixels",
404 default="SUSPECT",
405 )
406 numEdgeSuspect = pexConfig.Field(
407 dtype=int,
408 doc="Number of edge pixels to be flagged as untrustworthy.",
409 default=0,
410 )
411 edgeMaskLevel = pexConfig.ChoiceField(
412 dtype=str,
413 doc="Mask edge pixels in which coordinate frame: DETECTOR or AMP?",
414 default="DETECTOR",
415 allowed={
416 'DETECTOR': 'Mask only the edges of the full detector.',
417 'AMP': 'Mask edges of each amplifier.',
418 },
419 )
421 # Initial masking options.
422 doSetBadRegions = pexConfig.Field(
423 dtype=bool,
424 doc="Should we set the level of all BAD patches of the chip to the chip's average value?",
425 default=True,
426 )
427 badStatistic = pexConfig.ChoiceField(
428 dtype=str,
429 doc="How to estimate the average value for BAD regions.",
430 default='MEANCLIP',
431 allowed={
432 "MEANCLIP": "Correct using the (clipped) mean of good data",
433 "MEDIAN": "Correct using the median of the good data",
434 },
435 )
437 # Overscan subtraction configuration.
438 doOverscan = pexConfig.Field(
439 dtype=bool,
440 doc="Do overscan subtraction?",
441 default=True,
442 )
443 overscan = pexConfig.ConfigurableField(
444 target=OverscanCorrectionTask,
445 doc="Overscan subtraction task for image segments.",
446 )
448 overscanFitType = pexConfig.ChoiceField(
449 dtype=str,
450 doc="The method for fitting the overscan bias level.",
451 default='MEDIAN',
452 allowed={
453 "POLY": "Fit ordinary polynomial to the longest axis of the overscan region",
454 "CHEB": "Fit Chebyshev polynomial to the longest axis of the overscan region",
455 "LEG": "Fit Legendre polynomial to the longest axis of the overscan region",
456 "NATURAL_SPLINE": "Fit natural spline to the longest axis of the overscan region",
457 "CUBIC_SPLINE": "Fit cubic spline to the longest axis of the overscan region",
458 "AKIMA_SPLINE": "Fit Akima spline to the longest axis of the overscan region",
459 "MEAN": "Correct using the mean of the overscan region",
460 "MEANCLIP": "Correct using a clipped mean of the overscan region",
461 "MEDIAN": "Correct using the median of the overscan region",
462 "MEDIAN_PER_ROW": "Correct using the median per row of the overscan region",
463 },
464 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
465 " This option will no longer be used, and will be removed after v20.")
466 )
467 overscanOrder = pexConfig.Field(
468 dtype=int,
469 doc=("Order of polynomial or to fit if overscan fit type is a polynomial, "
470 "or number of spline knots if overscan fit type is a spline."),
471 default=1,
472 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
473 " This option will no longer be used, and will be removed after v20.")
474 )
475 overscanNumSigmaClip = pexConfig.Field(
476 dtype=float,
477 doc="Rejection threshold (sigma) for collapsing overscan before fit",
478 default=3.0,
479 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
480 " This option will no longer be used, and will be removed after v20.")
481 )
482 overscanIsInt = pexConfig.Field(
483 dtype=bool,
484 doc="Treat overscan as an integer image for purposes of overscan.FitType=MEDIAN"
485 " and overscan.FitType=MEDIAN_PER_ROW.",
486 default=True,
487 deprecated=("Please configure overscan via the OverscanCorrectionConfig interface."
488 " This option will no longer be used, and will be removed after v20.")
489 )
490 # These options do not get deprecated, as they define how we slice up the image data.
491 overscanNumLeadingColumnsToSkip = pexConfig.Field(
492 dtype=int,
493 doc="Number of columns to skip in overscan, i.e. those closest to amplifier",
494 default=0,
495 )
496 overscanNumTrailingColumnsToSkip = pexConfig.Field(
497 dtype=int,
498 doc="Number of columns to skip in overscan, i.e. those farthest from amplifier",
499 default=0,
500 )
501 overscanMaxDev = pexConfig.Field( 501 ↛ exitline 501 didn't jump to the function exit
502 dtype=float,
503 doc="Maximum deviation from the median for overscan",
504 default=1000.0, check=lambda x: x > 0
505 )
506 overscanBiasJump = pexConfig.Field(
507 dtype=bool,
508 doc="Fit the overscan in a piecewise-fashion to correct for bias jumps?",
509 default=False,
510 )
511 overscanBiasJumpKeyword = pexConfig.Field(
512 dtype=str,
513 doc="Header keyword containing information about devices.",
514 default="NO_SUCH_KEY",
515 )
516 overscanBiasJumpDevices = pexConfig.ListField(
517 dtype=str,
518 doc="List of devices that need piecewise overscan correction.",
519 default=(),
520 )
521 overscanBiasJumpLocation = pexConfig.Field(
522 dtype=int,
523 doc="Location of bias jump along y-axis.",
524 default=0,
525 )
527 # Amplifier to CCD assembly configuration
528 doAssembleCcd = pexConfig.Field(
529 dtype=bool,
530 default=True,
531 doc="Assemble amp-level exposures into a ccd-level exposure?"
532 )
533 assembleCcd = pexConfig.ConfigurableField(
534 target=AssembleCcdTask,
535 doc="CCD assembly task",
536 )
538 # General calibration configuration.
539 doAssembleIsrExposures = pexConfig.Field(
540 dtype=bool,
541 default=False,
542 doc="Assemble amp-level calibration exposures into ccd-level exposure?"
543 )
544 doTrimToMatchCalib = pexConfig.Field(
545 dtype=bool,
546 default=False,
547 doc="Trim raw data to match calibration bounding boxes?"
548 )
550 # Bias subtraction.
551 doBias = pexConfig.Field(
552 dtype=bool,
553 doc="Apply bias frame correction?",
554 default=True,
555 )
556 biasDataProductName = pexConfig.Field(
557 dtype=str,
558 doc="Name of the bias data product",
559 default="bias",
560 )
561 doBiasBeforeOverscan = pexConfig.Field(
562 dtype=bool,
563 doc="Reverse order of overscan and bias correction.",
564 default=False
565 )
567 # Variance construction
568 doVariance = pexConfig.Field(
569 dtype=bool,
570 doc="Calculate variance?",
571 default=True
572 )
573 gain = pexConfig.Field(
574 dtype=float,
575 doc="The gain to use if no Detector is present in the Exposure (ignored if NaN)",
576 default=float("NaN"),
577 )
578 readNoise = pexConfig.Field(
579 dtype=float,
580 doc="The read noise to use if no Detector is present in the Exposure",
581 default=0.0,
582 )
583 doEmpiricalReadNoise = pexConfig.Field(
584 dtype=bool,
585 default=False,
586 doc="Calculate empirical read noise instead of value from AmpInfo data?"
587 )
588 usePtcReadNoise = pexConfig.Field(
589 dtype=bool,
590 default=False,
591 doc="Use readnoise values from the Photon Transfer Curve?"
592 )
593 # Linearization.
594 doLinearize = pexConfig.Field(
595 dtype=bool,
596 doc="Correct for nonlinearity of the detector's response?",
597 default=True,
598 )
600 # Crosstalk.
601 doCrosstalk = pexConfig.Field(
602 dtype=bool,
603 doc="Apply intra-CCD crosstalk correction?",
604 default=False,
605 )
606 doCrosstalkBeforeAssemble = pexConfig.Field(
607 dtype=bool,
608 doc="Apply crosstalk correction before CCD assembly, and before trimming?",
609 default=False,
610 )
611 crosstalk = pexConfig.ConfigurableField(
612 target=CrosstalkTask,
613 doc="Intra-CCD crosstalk correction",
614 )
616 # Masking options.
617 doDefect = pexConfig.Field(
618 dtype=bool,
619 doc="Apply correction for CCD defects, e.g. hot pixels?",
620 default=True,
621 )
622 doNanMasking = pexConfig.Field(
623 dtype=bool,
624 doc="Mask non-finite (NAN, inf) pixels?",
625 default=True,
626 )
627 doWidenSaturationTrails = pexConfig.Field(
628 dtype=bool,
629 doc="Widen bleed trails based on their width?",
630 default=True
631 )
633 # Brighter-Fatter correction.
634 doBrighterFatter = pexConfig.Field(
635 dtype=bool,
636 default=False,
637 doc="Apply the brighter-fatter correction?"
638 )
639 brighterFatterLevel = pexConfig.ChoiceField(
640 dtype=str,
641 default="DETECTOR",
642 doc="The level at which to correct for brighter-fatter.",
643 allowed={
644 "AMP": "Every amplifier treated separately.",
645 "DETECTOR": "One kernel per detector",
646 }
647 )
648 brighterFatterMaxIter = pexConfig.Field(
649 dtype=int,
650 default=10,
651 doc="Maximum number of iterations for the brighter-fatter correction"
652 )
653 brighterFatterThreshold = pexConfig.Field(
654 dtype=float,
655 default=1000,
656 doc="Threshold used to stop iterating the brighter-fatter correction. It is the "
657 "absolute value of the difference between the current corrected image and the one "
658 "from the previous iteration summed over all the pixels."
659 )
660 brighterFatterApplyGain = pexConfig.Field(
661 dtype=bool,
662 default=True,
663 doc="Should the gain be applied when applying the brighter-fatter correction?"
664 )
665 brighterFatterMaskListToInterpolate = pexConfig.ListField(
666 dtype=str,
667 doc="List of mask planes that should be interpolated over when applying the brighter-fatter "
668 "correction.",
669 default=["SAT", "BAD", "NO_DATA", "UNMASKEDNAN"],
670 )
671 brighterFatterMaskGrowSize = pexConfig.Field(
672 dtype=int,
673 default=0,
674 doc="Number of pixels to grow the masks listed in config.brighterFatterMaskListToInterpolate "
675 "when brighter-fatter correction is applied."
676 )
678 # Dark subtraction.
679 doDark = pexConfig.Field(
680 dtype=bool,
681 doc="Apply dark frame correction?",
682 default=True,
683 )
684 darkDataProductName = pexConfig.Field(
685 dtype=str,
686 doc="Name of the dark data product",
687 default="dark",
688 )
690 # Camera-specific stray light removal.
691 doStrayLight = pexConfig.Field(
692 dtype=bool,
693 doc="Subtract stray light in the y-band (due to encoder LEDs)?",
694 default=False,
695 )
696 strayLight = pexConfig.ConfigurableField(
697 target=StrayLightTask,
698 doc="y-band stray light correction"
699 )
701 # Flat correction.
702 doFlat = pexConfig.Field(
703 dtype=bool,
704 doc="Apply flat field correction?",
705 default=True,
706 )
707 flatDataProductName = pexConfig.Field(
708 dtype=str,
709 doc="Name of the flat data product",
710 default="flat",
711 )
712 flatScalingType = pexConfig.ChoiceField(
713 dtype=str,
714 doc="The method for scaling the flat on the fly.",
715 default='USER',
716 allowed={
717 "USER": "Scale by flatUserScale",
718 "MEAN": "Scale by the inverse of the mean",
719 "MEDIAN": "Scale by the inverse of the median",
720 },
721 )
722 flatUserScale = pexConfig.Field(
723 dtype=float,
724 doc="If flatScalingType is 'USER' then scale flat by this amount; ignored otherwise",
725 default=1.0,
726 )
727 doTweakFlat = pexConfig.Field(
728 dtype=bool,
729 doc="Tweak flats to match observed amplifier ratios?",
730 default=False
731 )
733 # Amplifier normalization based on gains instead of using flats configuration.
734 doApplyGains = pexConfig.Field(
735 dtype=bool,
736 doc="Correct the amplifiers for their gains instead of applying flat correction",
737 default=False,
738 )
739 usePtcGains = pexConfig.Field(
740 dtype=bool,
741 doc="Use the gain values from the Photon Transfer Curve?",
742 default=False,
743 )
744 normalizeGains = pexConfig.Field(
745 dtype=bool,
746 doc="Normalize all the amplifiers in each CCD to have the same median value.",
747 default=False,
748 )
750 # Fringe correction.
751 doFringe = pexConfig.Field(
752 dtype=bool,
753 doc="Apply fringe correction?",
754 default=True,
755 )
756 fringe = pexConfig.ConfigurableField(
757 target=FringeTask,
758 doc="Fringe subtraction task",
759 )
760 fringeAfterFlat = pexConfig.Field(
761 dtype=bool,
762 doc="Do fringe subtraction after flat-fielding?",
763 default=True,
764 )
766 # Initial CCD-level background statistics options.
767 doMeasureBackground = pexConfig.Field(
768 dtype=bool,
769 doc="Measure the background level on the reduced image?",
770 default=False,
771 )
773 # Camera-specific masking configuration.
774 doCameraSpecificMasking = pexConfig.Field(
775 dtype=bool,
776 doc="Mask camera-specific bad regions?",
777 default=False,
778 )
779 masking = pexConfig.ConfigurableField(
780 target=MaskingTask,
781 doc="Masking task."
782 )
784 # Interpolation options.
786 doInterpolate = pexConfig.Field(
787 dtype=bool,
788 doc="Interpolate masked pixels?",
789 default=True,
790 )
791 doSaturationInterpolation = pexConfig.Field(
792 dtype=bool,
793 doc="Perform interpolation over pixels masked as saturated?"
794 " NB: This is independent of doSaturation; if that is False this plane"
795 " will likely be blank, resulting in a no-op here.",
796 default=True,
797 )
798 doNanInterpolation = pexConfig.Field(
799 dtype=bool,
800 doc="Perform interpolation over pixels masked as NaN?"
801 " NB: This is independent of doNanMasking; if that is False this plane"
802 " will likely be blank, resulting in a no-op here.",
803 default=True,
804 )
805 doNanInterpAfterFlat = pexConfig.Field(
806 dtype=bool,
807 doc=("If True, ensure we interpolate NaNs after flat-fielding, even if we "
808 "also have to interpolate them before flat-fielding."),
809 default=False,
810 )
811 maskListToInterpolate = pexConfig.ListField(
812 dtype=str,
813 doc="List of mask planes that should be interpolated.",
814 default=['SAT', 'BAD'],
815 )
816 doSaveInterpPixels = pexConfig.Field(
817 dtype=bool,
818 doc="Save a copy of the pre-interpolated pixel values?",
819 default=False,
820 )
822 # Default photometric calibration options.
823 fluxMag0T1 = pexConfig.DictField(
824 keytype=str,
825 itemtype=float,
826 doc="The approximate flux of a zero-magnitude object in a one-second exposure, per filter.",
827 default=dict((f, pow(10.0, 0.4*m)) for f, m in (("Unknown", 28.0),
828 ))
829 )
830 defaultFluxMag0T1 = pexConfig.Field(
831 dtype=float,
832 doc="Default value for fluxMag0T1 (for an unrecognized filter).",
833 default=pow(10.0, 0.4*28.0)
834 )
836 # Vignette correction configuration.
837 doVignette = pexConfig.Field(
838 dtype=bool,
839 doc="Apply vignetting parameters?",
840 default=False,
841 )
842 vignette = pexConfig.ConfigurableField(
843 target=VignetteTask,
844 doc="Vignetting task.",
845 )
847 # Transmission curve configuration.
848 doAttachTransmissionCurve = pexConfig.Field(
849 dtype=bool,
850 default=False,
851 doc="Construct and attach a wavelength-dependent throughput curve for this CCD image?"
852 )
853 doUseOpticsTransmission = pexConfig.Field(
854 dtype=bool,
855 default=True,
856 doc="Load and use transmission_optics (if doAttachTransmissionCurve is True)?"
857 )
858 doUseFilterTransmission = pexConfig.Field(
859 dtype=bool,
860 default=True,
861 doc="Load and use transmission_filter (if doAttachTransmissionCurve is True)?"
862 )
863 doUseSensorTransmission = pexConfig.Field(
864 dtype=bool,
865 default=True,
866 doc="Load and use transmission_sensor (if doAttachTransmissionCurve is True)?"
867 )
868 doUseAtmosphereTransmission = pexConfig.Field(
869 dtype=bool,
870 default=True,
871 doc="Load and use transmission_atmosphere (if doAttachTransmissionCurve is True)?"
872 )
874 # Illumination correction.
875 doIlluminationCorrection = pexConfig.Field(
876 dtype=bool,
877 default=False,
878 doc="Perform illumination correction?"
879 )
880 illuminationCorrectionDataProductName = pexConfig.Field(
881 dtype=str,
882 doc="Name of the illumination correction data product.",
883 default="illumcor",
884 )
885 illumScale = pexConfig.Field(
886 dtype=float,
887 doc="Scale factor for the illumination correction.",
888 default=1.0,
889 )
890 illumFilters = pexConfig.ListField(
891 dtype=str,
892 default=[],
893 doc="Only perform illumination correction for these filters."
894 )
896 # Write the outputs to disk. If ISR is run as a subtask, this may not be needed.
897 doWrite = pexConfig.Field(
898 dtype=bool,
899 doc="Persist postISRCCD?",
900 default=True,
901 )
903 def validate(self):
904 super().validate()
905 if self.doFlat and self.doApplyGains:
906 raise ValueError("You may not specify both doFlat and doApplyGains")
907 if self.doBiasBeforeOverscan and self.doTrimToMatchCalib:
908 raise ValueError("You may not specify both doBiasBeforeOverscan and doTrimToMatchCalib")
909 if self.doSaturationInterpolation and self.saturatedMaskName not in self.maskListToInterpolate:
910 self.maskListToInterpolate.append(self.saturatedMaskName)
911 if not self.doSaturationInterpolation and self.saturatedMaskName in self.maskListToInterpolate:
912 self.maskListToInterpolate.remove(self.saturatedMaskName)
913 if self.doNanInterpolation and "UNMASKEDNAN" not in self.maskListToInterpolate:
914 self.maskListToInterpolate.append("UNMASKEDNAN")
917class IsrTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
918 """Apply common instrument signature correction algorithms to a raw frame.
920 The process for correcting imaging data is very similar from
921 camera to camera. This task provides a vanilla implementation of
922 doing these corrections, including the ability to turn certain
923 corrections off if they are not needed. The inputs to the primary
924 method, `run()`, are a raw exposure to be corrected and the
925 calibration data products. The raw input is a single chip sized
926 mosaic of all amps including overscans and other non-science
927 pixels. The method `runDataRef()` identifies and defines the
928 calibration data products, and is intended for use by a
929 `lsst.pipe.base.cmdLineTask.CmdLineTask` and takes as input only a
930 `daf.persistence.butlerSubset.ButlerDataRef`. This task may be
931 subclassed for different camera, although the most camera specific
932 methods have been split into subtasks that can be redirected
933 appropriately.
935 The __init__ method sets up the subtasks for ISR processing, using
936 the defaults from `lsst.ip.isr`.
938 Parameters
939 ----------
940 args : `list`
941 Positional arguments passed to the Task constructor. None used at this time.
942 kwargs : `dict`, optional
943 Keyword arguments passed on to the Task constructor. None used at this time.
944 """
945 ConfigClass = IsrTaskConfig
946 _DefaultName = "isr"
948 def __init__(self, **kwargs):
949 super().__init__(**kwargs)
950 self.makeSubtask("assembleCcd")
951 self.makeSubtask("crosstalk")
952 self.makeSubtask("strayLight")
953 self.makeSubtask("fringe")
954 self.makeSubtask("masking")
955 self.makeSubtask("overscan")
956 self.makeSubtask("vignette")
958 def runQuantum(self, butlerQC, inputRefs, outputRefs):
959 inputs = butlerQC.get(inputRefs)
961 try:
962 inputs['detectorNum'] = inputRefs.ccdExposure.dataId['detector']
963 except Exception as e:
964 raise ValueError("Failure to find valid detectorNum value for Dataset %s: %s." %
965 (inputRefs, e))
967 inputs['isGen3'] = True
969 detector = inputs['ccdExposure'].getDetector()
971 if self.config.doCrosstalk is True:
972 # Crosstalk sources need to be defined by the pipeline
973 # yaml if they exist.
974 if 'crosstalk' in inputs and inputs['crosstalk'] is not None:
975 if not isinstance(inputs['crosstalk'], CrosstalkCalib):
976 inputs['crosstalk'] = CrosstalkCalib.fromTable(inputs['crosstalk'])
977 else:
978 coeffVector = (self.config.crosstalk.crosstalkValues
979 if self.config.crosstalk.useConfigCoefficients else None)
980 crosstalkCalib = CrosstalkCalib().fromDetector(detector, coeffVector=coeffVector)
981 inputs['crosstalk'] = crosstalkCalib
982 if inputs['crosstalk'].interChip and len(inputs['crosstalk'].interChip) > 0:
983 if 'crosstalkSources' not in inputs:
984 self.log.warning("No crosstalkSources found for chip with interChip terms!")
986 if self.doLinearize(detector) is True:
987 if 'linearizer' in inputs:
988 if isinstance(inputs['linearizer'], dict):
989 linearizer = linearize.Linearizer(detector=detector, log=self.log)
990 linearizer.fromYaml(inputs['linearizer'])
991 self.log.warning("Dictionary linearizers will be deprecated in DM-28741.")
992 elif isinstance(inputs['linearizer'], numpy.ndarray):
993 linearizer = linearize.Linearizer(table=inputs.get('linearizer', None),
994 detector=detector,
995 log=self.log)
996 self.log.warning("Bare lookup table linearizers will be deprecated in DM-28741.")
997 else:
998 linearizer = inputs['linearizer']
999 linearizer.log = self.log
1000 inputs['linearizer'] = linearizer
1001 else:
1002 inputs['linearizer'] = linearize.Linearizer(detector=detector, log=self.log)
1003 self.log.warning("Constructing linearizer from cameraGeom information.")
1005 if self.config.doDefect is True:
1006 if "defects" in inputs and inputs['defects'] is not None:
1007 # defects is loaded as a BaseCatalog with columns x0, y0, width, height.
1008 # masking expects a list of defects defined by their bounding box
1009 if not isinstance(inputs["defects"], Defects):
1010 inputs["defects"] = Defects.fromTable(inputs["defects"])
1012 # Load the correct style of brighter-fatter kernel, and repack
1013 # the information as a numpy array.
1014 if self.config.doBrighterFatter:
1015 brighterFatterKernel = inputs.pop('newBFKernel', None)
1016 if brighterFatterKernel is None:
1017 brighterFatterKernel = inputs.get('bfKernel', None)
1019 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1020 # This is a ISR calib kernel
1021 detName = detector.getName()
1022 level = brighterFatterKernel.level
1024 # This is expected to be a dictionary of amp-wise gains.
1025 inputs['bfGains'] = brighterFatterKernel.gain
1026 if self.config.brighterFatterLevel == 'DETECTOR':
1027 if level == 'DETECTOR':
1028 if detName in brighterFatterKernel.detKernels:
1029 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1030 else:
1031 raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1032 elif level == 'AMP':
1033 self.log.warning("Making DETECTOR level kernel from AMP based brighter "
1034 "fatter kernels.")
1035 brighterFatterKernel.makeDetectorKernelFromAmpwiseKernels(detName)
1036 inputs['bfKernel'] = brighterFatterKernel.detKernels[detName]
1037 elif self.config.brighterFatterLevel == 'AMP':
1038 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1040 if self.config.doFringe is True and self.fringe.checkFilter(inputs['ccdExposure']):
1041 expId = inputs['ccdExposure'].getInfo().getVisitInfo().getExposureId()
1042 inputs['fringes'] = self.fringe.loadFringes(inputs['fringes'],
1043 expId=expId,
1044 assembler=self.assembleCcd
1045 if self.config.doAssembleIsrExposures else None)
1046 else:
1047 inputs['fringes'] = pipeBase.Struct(fringes=None)
1049 if self.config.doStrayLight is True and self.strayLight.checkFilter(inputs['ccdExposure']):
1050 if 'strayLightData' not in inputs:
1051 inputs['strayLightData'] = None
1053 outputs = self.run(**inputs)
1054 butlerQC.put(outputs, outputRefs)
1056 def readIsrData(self, dataRef, rawExposure):
1057 """Retrieve necessary frames for instrument signature removal.
1059 Pre-fetching all required ISR data products limits the IO
1060 required by the ISR. Any conflict between the calibration data
1061 available and that needed for ISR is also detected prior to
1062 doing processing, allowing it to fail quickly.
1064 Parameters
1065 ----------
1066 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1067 Butler reference of the detector data to be processed
1068 rawExposure : `afw.image.Exposure`
1069 The raw exposure that will later be corrected with the
1070 retrieved calibration data; should not be modified in this
1071 method.
1073 Returns
1074 -------
1075 result : `lsst.pipe.base.Struct`
1076 Result struct with components (which may be `None`):
1077 - ``bias``: bias calibration frame (`afw.image.Exposure`)
1078 - ``linearizer``: functor for linearization (`ip.isr.linearize.LinearizeBase`)
1079 - ``crosstalkSources``: list of possible crosstalk sources (`list`)
1080 - ``dark``: dark calibration frame (`afw.image.Exposure`)
1081 - ``flat``: flat calibration frame (`afw.image.Exposure`)
1082 - ``bfKernel``: Brighter-Fatter kernel (`numpy.ndarray`)
1083 - ``defects``: list of defects (`lsst.ip.isr.Defects`)
1084 - ``fringes``: `lsst.pipe.base.Struct` with components:
1085 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1086 - ``seed``: random seed derived from the ccdExposureId for random
1087 number generator (`uint32`).
1088 - ``opticsTransmission``: `lsst.afw.image.TransmissionCurve`
1089 A ``TransmissionCurve`` that represents the throughput of the optics,
1090 to be evaluated in focal-plane coordinates.
1091 - ``filterTransmission`` : `lsst.afw.image.TransmissionCurve`
1092 A ``TransmissionCurve`` that represents the throughput of the filter
1093 itself, to be evaluated in focal-plane coordinates.
1094 - ``sensorTransmission`` : `lsst.afw.image.TransmissionCurve`
1095 A ``TransmissionCurve`` that represents the throughput of the sensor
1096 itself, to be evaluated in post-assembly trimmed detector coordinates.
1097 - ``atmosphereTransmission`` : `lsst.afw.image.TransmissionCurve`
1098 A ``TransmissionCurve`` that represents the throughput of the
1099 atmosphere, assumed to be spatially constant.
1100 - ``strayLightData`` : `object`
1101 An opaque object containing calibration information for
1102 stray-light correction. If `None`, no correction will be
1103 performed.
1104 - ``illumMaskedImage`` : illumination correction image (`lsst.afw.image.MaskedImage`)
1106 Raises
1107 ------
1108 NotImplementedError :
1109 Raised if a per-amplifier brighter-fatter kernel is requested by the configuration.
1110 """
1111 try:
1112 dateObs = rawExposure.getInfo().getVisitInfo().getDate()
1113 dateObs = dateObs.toPython().isoformat()
1114 except RuntimeError:
1115 self.log.warning("Unable to identify dateObs for rawExposure.")
1116 dateObs = None
1118 ccd = rawExposure.getDetector()
1119 filterLabel = rawExposure.getFilterLabel()
1120 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1121 rawExposure.mask.addMaskPlane("UNMASKEDNAN") # needed to match pre DM-15862 processing.
1122 biasExposure = (self.getIsrExposure(dataRef, self.config.biasDataProductName)
1123 if self.config.doBias else None)
1124 # immediate=True required for functors and linearizers are functors; see ticket DM-6515
1125 linearizer = (dataRef.get("linearizer", immediate=True)
1126 if self.doLinearize(ccd) else None)
1127 if linearizer is not None and not isinstance(linearizer, numpy.ndarray):
1128 linearizer.log = self.log
1129 if isinstance(linearizer, numpy.ndarray):
1130 linearizer = linearize.Linearizer(table=linearizer, detector=ccd)
1132 crosstalkCalib = None
1133 if self.config.doCrosstalk:
1134 try:
1135 crosstalkCalib = dataRef.get("crosstalk", immediate=True)
1136 except NoResults:
1137 coeffVector = (self.config.crosstalk.crosstalkValues
1138 if self.config.crosstalk.useConfigCoefficients else None)
1139 crosstalkCalib = CrosstalkCalib().fromDetector(ccd, coeffVector=coeffVector)
1140 crosstalkSources = (self.crosstalk.prepCrosstalk(dataRef, crosstalkCalib)
1141 if self.config.doCrosstalk else None)
1143 darkExposure = (self.getIsrExposure(dataRef, self.config.darkDataProductName)
1144 if self.config.doDark else None)
1145 flatExposure = (self.getIsrExposure(dataRef, self.config.flatDataProductName,
1146 dateObs=dateObs)
1147 if self.config.doFlat else None)
1149 brighterFatterKernel = None
1150 brighterFatterGains = None
1151 if self.config.doBrighterFatter is True:
1152 try:
1153 # Use the new-style cp_pipe version of the kernel if it exists
1154 # If using a new-style kernel, always use the self-consistent
1155 # gains, i.e. the ones inside the kernel object itself
1156 brighterFatterKernel = dataRef.get("brighterFatterKernel")
1157 brighterFatterGains = brighterFatterKernel.gain
1158 self.log.info("New style brighter-fatter kernel (brighterFatterKernel) loaded")
1159 except NoResults:
1160 try: # Fall back to the old-style numpy-ndarray style kernel if necessary.
1161 brighterFatterKernel = dataRef.get("bfKernel")
1162 self.log.info("Old style brighter-fatter kernel (bfKernel) loaded")
1163 except NoResults:
1164 brighterFatterKernel = None
1165 if brighterFatterKernel is not None and not isinstance(brighterFatterKernel, numpy.ndarray):
1166 # If the kernel is not an ndarray, it's the cp_pipe version
1167 # so extract the kernel for this detector, or raise an error
1168 if self.config.brighterFatterLevel == 'DETECTOR':
1169 if brighterFatterKernel.detKernels:
1170 brighterFatterKernel = brighterFatterKernel.detKernels[ccd.getName()]
1171 else:
1172 raise RuntimeError("Failed to extract kernel from new-style BF kernel.")
1173 else:
1174 # TODO DM-15631 for implementing this
1175 raise NotImplementedError("Per-amplifier brighter-fatter correction not implemented")
1177 defectList = (dataRef.get("defects")
1178 if self.config.doDefect else None)
1179 expId = rawExposure.getInfo().getVisitInfo().getExposureId()
1180 fringeStruct = (self.fringe.readFringes(dataRef, expId=expId, assembler=self.assembleCcd
1181 if self.config.doAssembleIsrExposures else None)
1182 if self.config.doFringe and self.fringe.checkFilter(rawExposure)
1183 else pipeBase.Struct(fringes=None))
1185 if self.config.doAttachTransmissionCurve:
1186 opticsTransmission = (dataRef.get("transmission_optics")
1187 if self.config.doUseOpticsTransmission else None)
1188 filterTransmission = (dataRef.get("transmission_filter")
1189 if self.config.doUseFilterTransmission else None)
1190 sensorTransmission = (dataRef.get("transmission_sensor")
1191 if self.config.doUseSensorTransmission else None)
1192 atmosphereTransmission = (dataRef.get("transmission_atmosphere")
1193 if self.config.doUseAtmosphereTransmission else None)
1194 else:
1195 opticsTransmission = None
1196 filterTransmission = None
1197 sensorTransmission = None
1198 atmosphereTransmission = None
1200 if self.config.doStrayLight:
1201 strayLightData = self.strayLight.readIsrData(dataRef, rawExposure)
1202 else:
1203 strayLightData = None
1205 illumMaskedImage = (self.getIsrExposure(dataRef,
1206 self.config.illuminationCorrectionDataProductName).getMaskedImage()
1207 if (self.config.doIlluminationCorrection
1208 and physicalFilter in self.config.illumFilters)
1209 else None)
1211 # Struct should include only kwargs to run()
1212 return pipeBase.Struct(bias=biasExposure,
1213 linearizer=linearizer,
1214 crosstalk=crosstalkCalib,
1215 crosstalkSources=crosstalkSources,
1216 dark=darkExposure,
1217 flat=flatExposure,
1218 bfKernel=brighterFatterKernel,
1219 bfGains=brighterFatterGains,
1220 defects=defectList,
1221 fringes=fringeStruct,
1222 opticsTransmission=opticsTransmission,
1223 filterTransmission=filterTransmission,
1224 sensorTransmission=sensorTransmission,
1225 atmosphereTransmission=atmosphereTransmission,
1226 strayLightData=strayLightData,
1227 illumMaskedImage=illumMaskedImage
1228 )
1230 @pipeBase.timeMethod
1231 def run(self, ccdExposure, *, camera=None, bias=None, linearizer=None,
1232 crosstalk=None, crosstalkSources=None,
1233 dark=None, flat=None, ptc=None, bfKernel=None, bfGains=None, defects=None,
1234 fringes=pipeBase.Struct(fringes=None), opticsTransmission=None, filterTransmission=None,
1235 sensorTransmission=None, atmosphereTransmission=None,
1236 detectorNum=None, strayLightData=None, illumMaskedImage=None,
1237 isGen3=False,
1238 ):
1239 """Perform instrument signature removal on an exposure.
1241 Steps included in the ISR processing, in order performed, are:
1242 - saturation and suspect pixel masking
1243 - overscan subtraction
1244 - CCD assembly of individual amplifiers
1245 - bias subtraction
1246 - variance image construction
1247 - linearization of non-linear response
1248 - crosstalk masking
1249 - brighter-fatter correction
1250 - dark subtraction
1251 - fringe correction
1252 - stray light subtraction
1253 - flat correction
1254 - masking of known defects and camera specific features
1255 - vignette calculation
1256 - appending transmission curve and distortion model
1258 Parameters
1259 ----------
1260 ccdExposure : `lsst.afw.image.Exposure`
1261 The raw exposure that is to be run through ISR. The
1262 exposure is modified by this method.
1263 camera : `lsst.afw.cameraGeom.Camera`, optional
1264 The camera geometry for this exposure. Required if ``isGen3`` is
1265 `True` and one or more of ``ccdExposure``, ``bias``, ``dark``, or
1266 ``flat`` does not have an associated detector.
1267 bias : `lsst.afw.image.Exposure`, optional
1268 Bias calibration frame.
1269 linearizer : `lsst.ip.isr.linearize.LinearizeBase`, optional
1270 Functor for linearization.
1271 crosstalk : `lsst.ip.isr.crosstalk.CrosstalkCalib`, optional
1272 Calibration for crosstalk.
1273 crosstalkSources : `list`, optional
1274 List of possible crosstalk sources.
1275 dark : `lsst.afw.image.Exposure`, optional
1276 Dark calibration frame.
1277 flat : `lsst.afw.image.Exposure`, optional
1278 Flat calibration frame.
1279 ptc : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
1280 Photon transfer curve dataset, with, e.g., gains
1281 and read noise.
1282 bfKernel : `numpy.ndarray`, optional
1283 Brighter-fatter kernel.
1284 bfGains : `dict` of `float`, optional
1285 Gains used to override the detector's nominal gains for the
1286 brighter-fatter correction. A dict keyed by amplifier name for
1287 the detector in question.
1288 defects : `lsst.ip.isr.Defects`, optional
1289 List of defects.
1290 fringes : `lsst.pipe.base.Struct`, optional
1291 Struct containing the fringe correction data, with
1292 elements:
1293 - ``fringes``: fringe calibration frame (`afw.image.Exposure`)
1294 - ``seed``: random seed derived from the ccdExposureId for random
1295 number generator (`uint32`)
1296 opticsTransmission: `lsst.afw.image.TransmissionCurve`, optional
1297 A ``TransmissionCurve`` that represents the throughput of the optics,
1298 to be evaluated in focal-plane coordinates.
1299 filterTransmission : `lsst.afw.image.TransmissionCurve`
1300 A ``TransmissionCurve`` that represents the throughput of the filter
1301 itself, to be evaluated in focal-plane coordinates.
1302 sensorTransmission : `lsst.afw.image.TransmissionCurve`
1303 A ``TransmissionCurve`` that represents the throughput of the sensor
1304 itself, to be evaluated in post-assembly trimmed detector coordinates.
1305 atmosphereTransmission : `lsst.afw.image.TransmissionCurve`
1306 A ``TransmissionCurve`` that represents the throughput of the
1307 atmosphere, assumed to be spatially constant.
1308 detectorNum : `int`, optional
1309 The integer number for the detector to process.
1310 isGen3 : bool, optional
1311 Flag this call to run() as using the Gen3 butler environment.
1312 strayLightData : `object`, optional
1313 Opaque object containing calibration information for stray-light
1314 correction. If `None`, no correction will be performed.
1315 illumMaskedImage : `lsst.afw.image.MaskedImage`, optional
1316 Illumination correction image.
1318 Returns
1319 -------
1320 result : `lsst.pipe.base.Struct`
1321 Result struct with component:
1322 - ``exposure`` : `afw.image.Exposure`
1323 The fully ISR corrected exposure.
1324 - ``outputExposure`` : `afw.image.Exposure`
1325 An alias for `exposure`
1326 - ``ossThumb`` : `numpy.ndarray`
1327 Thumbnail image of the exposure after overscan subtraction.
1328 - ``flattenedThumb`` : `numpy.ndarray`
1329 Thumbnail image of the exposure after flat-field correction.
1331 Raises
1332 ------
1333 RuntimeError
1334 Raised if a configuration option is set to True, but the
1335 required calibration data has not been specified.
1337 Notes
1338 -----
1339 The current processed exposure can be viewed by setting the
1340 appropriate lsstDebug entries in the `debug.display`
1341 dictionary. The names of these entries correspond to some of
1342 the IsrTaskConfig Boolean options, with the value denoting the
1343 frame to use. The exposure is shown inside the matching
1344 option check and after the processing of that step has
1345 finished. The steps with debug points are:
1347 doAssembleCcd
1348 doBias
1349 doCrosstalk
1350 doBrighterFatter
1351 doDark
1352 doFringe
1353 doStrayLight
1354 doFlat
1356 In addition, setting the "postISRCCD" entry displays the
1357 exposure after all ISR processing has finished.
1359 """
1361 if isGen3 is True:
1362 # Gen3 currently cannot automatically do configuration overrides.
1363 # DM-15257 looks to discuss this issue.
1364 # Configure input exposures;
1365 if detectorNum is None:
1366 raise RuntimeError("Must supply the detectorNum if running as Gen3.")
1368 ccdExposure = self.ensureExposure(ccdExposure, camera, detectorNum)
1369 bias = self.ensureExposure(bias, camera, detectorNum)
1370 dark = self.ensureExposure(dark, camera, detectorNum)
1371 flat = self.ensureExposure(flat, camera, detectorNum)
1372 else:
1373 if isinstance(ccdExposure, ButlerDataRef):
1374 return self.runDataRef(ccdExposure)
1376 ccd = ccdExposure.getDetector()
1377 filterLabel = ccdExposure.getFilterLabel()
1378 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
1380 if not ccd:
1381 assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd."
1382 ccd = [FakeAmp(ccdExposure, self.config)]
1384 # Validate Input
1385 if self.config.doBias and bias is None:
1386 raise RuntimeError("Must supply a bias exposure if config.doBias=True.")
1387 if self.doLinearize(ccd) and linearizer is None:
1388 raise RuntimeError("Must supply a linearizer if config.doLinearize=True for this detector.")
1389 if self.config.doBrighterFatter and bfKernel is None:
1390 raise RuntimeError("Must supply a kernel if config.doBrighterFatter=True.")
1391 if self.config.doDark and dark is None:
1392 raise RuntimeError("Must supply a dark exposure if config.doDark=True.")
1393 if self.config.doFlat and flat is None:
1394 raise RuntimeError("Must supply a flat exposure if config.doFlat=True.")
1395 if self.config.doDefect and defects is None:
1396 raise RuntimeError("Must supply defects if config.doDefect=True.")
1397 if (self.config.doFringe and physicalFilter in self.fringe.config.filters
1398 and fringes.fringes is None):
1399 # The `fringes` object needs to be a pipeBase.Struct, as
1400 # we use it as a `dict` for the parameters of
1401 # `FringeTask.run()`. The `fringes.fringes` `list` may
1402 # not be `None` if `doFringe=True`. Otherwise, raise.
1403 raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct.")
1404 if (self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters
1405 and illumMaskedImage is None):
1406 raise RuntimeError("Must supply an illumcor if config.doIlluminationCorrection=True.")
1408 # Begin ISR processing.
1409 if self.config.doConvertIntToFloat:
1410 self.log.info("Converting exposure to floating point values.")
1411 ccdExposure = self.convertIntToFloat(ccdExposure)
1413 if self.config.doBias and self.config.doBiasBeforeOverscan:
1414 self.log.info("Applying bias correction.")
1415 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1416 trimToFit=self.config.doTrimToMatchCalib)
1417 self.debugView(ccdExposure, "doBias")
1419 # Amplifier level processing.
1420 overscans = []
1421 for amp in ccd:
1422 # if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
1423 if ccdExposure.getBBox().contains(amp.getBBox()):
1424 # Check for fully masked bad amplifiers, and generate masks for SUSPECT and SATURATED values.
1425 badAmp = self.maskAmplifier(ccdExposure, amp, defects)
1427 if self.config.doOverscan and not badAmp:
1428 # Overscan correction on amp-by-amp basis.
1429 overscanResults = self.overscanCorrection(ccdExposure, amp)
1430 self.log.debug("Corrected overscan for amplifier %s.", amp.getName())
1431 if overscanResults is not None and \
1432 self.config.qa is not None and self.config.qa.saveStats is True:
1433 if isinstance(overscanResults.overscanFit, float):
1434 qaMedian = overscanResults.overscanFit
1435 qaStdev = float("NaN")
1436 else:
1437 qaStats = afwMath.makeStatistics(overscanResults.overscanFit,
1438 afwMath.MEDIAN | afwMath.STDEVCLIP)
1439 qaMedian = qaStats.getValue(afwMath.MEDIAN)
1440 qaStdev = qaStats.getValue(afwMath.STDEVCLIP)
1442 self.metadata.set(f"FIT MEDIAN {amp.getName()}", qaMedian)
1443 self.metadata.set(f"FIT STDEV {amp.getName()}", qaStdev)
1444 self.log.debug(" Overscan stats for amplifer %s: %f +/- %f",
1445 amp.getName(), qaMedian, qaStdev)
1447 # Residuals after overscan correction
1448 qaStatsAfter = afwMath.makeStatistics(overscanResults.overscanImage,
1449 afwMath.MEDIAN | afwMath.STDEVCLIP)
1450 qaMedianAfter = qaStatsAfter.getValue(afwMath.MEDIAN)
1451 qaStdevAfter = qaStatsAfter.getValue(afwMath.STDEVCLIP)
1453 self.metadata.set(f"RESIDUAL MEDIAN {amp.getName()}", qaMedianAfter)
1454 self.metadata.set(f"RESIDUAL STDEV {amp.getName()}", qaStdevAfter)
1455 self.log.debug(" Overscan stats for amplifer %s after correction: %f +/- %f",
1456 amp.getName(), qaMedianAfter, qaStdevAfter)
1458 ccdExposure.getMetadata().set('OVERSCAN', "Overscan corrected")
1459 else:
1460 if badAmp:
1461 self.log.warning("Amplifier %s is bad.", amp.getName())
1462 overscanResults = None
1464 overscans.append(overscanResults if overscanResults is not None else None)
1465 else:
1466 self.log.info("Skipped OSCAN for %s.", amp.getName())
1468 if self.config.doCrosstalk and self.config.doCrosstalkBeforeAssemble:
1469 self.log.info("Applying crosstalk correction.")
1470 self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1471 crosstalkSources=crosstalkSources, camera=camera)
1472 self.debugView(ccdExposure, "doCrosstalk")
1474 if self.config.doAssembleCcd:
1475 self.log.info("Assembling CCD from amplifiers.")
1476 ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
1478 if self.config.expectWcs and not ccdExposure.getWcs():
1479 self.log.warning("No WCS found in input exposure.")
1480 self.debugView(ccdExposure, "doAssembleCcd")
1482 ossThumb = None
1483 if self.config.qa.doThumbnailOss:
1484 ossThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1486 if self.config.doBias and not self.config.doBiasBeforeOverscan:
1487 self.log.info("Applying bias correction.")
1488 isrFunctions.biasCorrection(ccdExposure.getMaskedImage(), bias.getMaskedImage(),
1489 trimToFit=self.config.doTrimToMatchCalib)
1490 self.debugView(ccdExposure, "doBias")
1492 if self.config.doVariance:
1493 for amp, overscanResults in zip(ccd, overscans):
1494 if ccdExposure.getBBox().contains(amp.getBBox()):
1495 self.log.debug("Constructing variance map for amplifer %s.", amp.getName())
1496 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1497 if overscanResults is not None:
1498 self.updateVariance(ampExposure, amp,
1499 overscanImage=overscanResults.overscanImage,
1500 ptcDataset=ptc)
1501 else:
1502 self.updateVariance(ampExposure, amp,
1503 overscanImage=None,
1504 ptcDataset=ptc)
1505 if self.config.qa is not None and self.config.qa.saveStats is True:
1506 qaStats = afwMath.makeStatistics(ampExposure.getVariance(),
1507 afwMath.MEDIAN | afwMath.STDEVCLIP)
1508 self.metadata.set(f"ISR VARIANCE {amp.getName()} MEDIAN",
1509 qaStats.getValue(afwMath.MEDIAN))
1510 self.metadata.set(f"ISR VARIANCE {amp.getName()} STDEV",
1511 qaStats.getValue(afwMath.STDEVCLIP))
1512 self.log.debug(" Variance stats for amplifer %s: %f +/- %f.",
1513 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1514 qaStats.getValue(afwMath.STDEVCLIP))
1516 if self.doLinearize(ccd):
1517 self.log.info("Applying linearizer.")
1518 linearizer.applyLinearity(image=ccdExposure.getMaskedImage().getImage(),
1519 detector=ccd, log=self.log)
1521 if self.config.doCrosstalk and not self.config.doCrosstalkBeforeAssemble:
1522 self.log.info("Applying crosstalk correction.")
1523 self.crosstalk.run(ccdExposure, crosstalk=crosstalk,
1524 crosstalkSources=crosstalkSources, isTrimmed=True)
1525 self.debugView(ccdExposure, "doCrosstalk")
1527 # Masking block. Optionally mask known defects, NAN/inf pixels, widen trails, and do
1528 # anything else the camera needs. Saturated and suspect pixels have already been masked.
1529 if self.config.doDefect:
1530 self.log.info("Masking defects.")
1531 self.maskDefect(ccdExposure, defects)
1533 if self.config.numEdgeSuspect > 0:
1534 self.log.info("Masking edges as SUSPECT.")
1535 self.maskEdges(ccdExposure, numEdgePixels=self.config.numEdgeSuspect,
1536 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
1538 if self.config.doNanMasking:
1539 self.log.info("Masking non-finite (NAN, inf) value pixels.")
1540 self.maskNan(ccdExposure)
1542 if self.config.doWidenSaturationTrails:
1543 self.log.info("Widening saturation trails.")
1544 isrFunctions.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
1546 if self.config.doCameraSpecificMasking:
1547 self.log.info("Masking regions for camera specific reasons.")
1548 self.masking.run(ccdExposure)
1550 if self.config.doBrighterFatter:
1551 # We need to apply flats and darks before we can interpolate, and we
1552 # need to interpolate before we do B-F, but we do B-F without the
1553 # flats and darks applied so we can work in units of electrons or holes.
1554 # This context manager applies and then removes the darks and flats.
1555 #
1556 # We also do not want to interpolate values here, so operate on temporary
1557 # images so we can apply only the BF-correction and roll back the
1558 # interpolation.
1559 interpExp = ccdExposure.clone()
1560 with self.flatContext(interpExp, flat, dark):
1561 isrFunctions.interpolateFromMask(
1562 maskedImage=interpExp.getMaskedImage(),
1563 fwhm=self.config.fwhm,
1564 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1565 maskNameList=list(self.config.brighterFatterMaskListToInterpolate)
1566 )
1567 bfExp = interpExp.clone()
1569 self.log.info("Applying brighter-fatter correction using kernel type %s / gains %s.",
1570 type(bfKernel), type(bfGains))
1571 bfResults = isrFunctions.brighterFatterCorrection(bfExp, bfKernel,
1572 self.config.brighterFatterMaxIter,
1573 self.config.brighterFatterThreshold,
1574 self.config.brighterFatterApplyGain,
1575 bfGains)
1576 if bfResults[1] == self.config.brighterFatterMaxIter:
1577 self.log.warning("Brighter-fatter correction did not converge, final difference %f.",
1578 bfResults[0])
1579 else:
1580 self.log.info("Finished brighter-fatter correction in %d iterations.",
1581 bfResults[1])
1582 image = ccdExposure.getMaskedImage().getImage()
1583 bfCorr = bfExp.getMaskedImage().getImage()
1584 bfCorr -= interpExp.getMaskedImage().getImage()
1585 image += bfCorr
1587 # Applying the brighter-fatter correction applies a
1588 # convolution to the science image. At the edges this
1589 # convolution may not have sufficient valid pixels to
1590 # produce a valid correction. Mark pixels within the size
1591 # of the brighter-fatter kernel as EDGE to warn of this
1592 # fact.
1593 self.log.info("Ensuring image edges are masked as EDGE to the brighter-fatter kernel size.")
1594 self.maskEdges(ccdExposure, numEdgePixels=numpy.max(bfKernel.shape) // 2,
1595 maskPlane="EDGE")
1597 if self.config.brighterFatterMaskGrowSize > 0:
1598 self.log.info("Growing masks to account for brighter-fatter kernel convolution.")
1599 for maskPlane in self.config.brighterFatterMaskListToInterpolate:
1600 isrFunctions.growMasks(ccdExposure.getMask(),
1601 radius=self.config.brighterFatterMaskGrowSize,
1602 maskNameList=maskPlane,
1603 maskValue=maskPlane)
1605 self.debugView(ccdExposure, "doBrighterFatter")
1607 if self.config.doDark:
1608 self.log.info("Applying dark correction.")
1609 self.darkCorrection(ccdExposure, dark)
1610 self.debugView(ccdExposure, "doDark")
1612 if self.config.doFringe and not self.config.fringeAfterFlat:
1613 self.log.info("Applying fringe correction before flat.")
1614 self.fringe.run(ccdExposure, **fringes.getDict())
1615 self.debugView(ccdExposure, "doFringe")
1617 if self.config.doStrayLight and self.strayLight.check(ccdExposure):
1618 self.log.info("Checking strayLight correction.")
1619 self.strayLight.run(ccdExposure, strayLightData)
1620 self.debugView(ccdExposure, "doStrayLight")
1622 if self.config.doFlat:
1623 self.log.info("Applying flat correction.")
1624 self.flatCorrection(ccdExposure, flat)
1625 self.debugView(ccdExposure, "doFlat")
1627 if self.config.doApplyGains:
1628 self.log.info("Applying gain correction instead of flat.")
1629 if self.config.usePtcGains:
1630 self.log.info("Using gains from the Photon Transfer Curve.")
1631 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains,
1632 ptcGains=ptc.gain)
1633 else:
1634 isrFunctions.applyGains(ccdExposure, self.config.normalizeGains)
1636 if self.config.doFringe and self.config.fringeAfterFlat:
1637 self.log.info("Applying fringe correction after flat.")
1638 self.fringe.run(ccdExposure, **fringes.getDict())
1640 if self.config.doVignette:
1641 self.log.info("Constructing Vignette polygon.")
1642 self.vignettePolygon = self.vignette.run(ccdExposure)
1644 if self.config.vignette.doWriteVignettePolygon:
1645 self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
1647 if self.config.doAttachTransmissionCurve:
1648 self.log.info("Adding transmission curves.")
1649 isrFunctions.attachTransmissionCurve(ccdExposure, opticsTransmission=opticsTransmission,
1650 filterTransmission=filterTransmission,
1651 sensorTransmission=sensorTransmission,
1652 atmosphereTransmission=atmosphereTransmission)
1654 flattenedThumb = None
1655 if self.config.qa.doThumbnailFlattened:
1656 flattenedThumb = isrQa.makeThumbnail(ccdExposure, isrQaConfig=self.config.qa)
1658 if self.config.doIlluminationCorrection and physicalFilter in self.config.illumFilters:
1659 self.log.info("Performing illumination correction.")
1660 isrFunctions.illuminationCorrection(ccdExposure.getMaskedImage(),
1661 illumMaskedImage, illumScale=self.config.illumScale,
1662 trimToFit=self.config.doTrimToMatchCalib)
1664 preInterpExp = None
1665 if self.config.doSaveInterpPixels:
1666 preInterpExp = ccdExposure.clone()
1668 # Reset and interpolate bad pixels.
1669 #
1670 # Large contiguous bad regions (which should have the BAD mask
1671 # bit set) should have their values set to the image median.
1672 # This group should include defects and bad amplifiers. As the
1673 # area covered by these defects are large, there's little
1674 # reason to expect that interpolation would provide a more
1675 # useful value.
1676 #
1677 # Smaller defects can be safely interpolated after the larger
1678 # regions have had their pixel values reset. This ensures
1679 # that the remaining defects adjacent to bad amplifiers (as an
1680 # example) do not attempt to interpolate extreme values.
1681 if self.config.doSetBadRegions:
1682 badPixelCount, badPixelValue = isrFunctions.setBadRegions(ccdExposure)
1683 if badPixelCount > 0:
1684 self.log.info("Set %d BAD pixels to %f.", badPixelCount, badPixelValue)
1686 if self.config.doInterpolate:
1687 self.log.info("Interpolating masked pixels.")
1688 isrFunctions.interpolateFromMask(
1689 maskedImage=ccdExposure.getMaskedImage(),
1690 fwhm=self.config.fwhm,
1691 growSaturatedFootprints=self.config.growSaturationFootprintSize,
1692 maskNameList=list(self.config.maskListToInterpolate)
1693 )
1695 self.roughZeroPoint(ccdExposure)
1697 if self.config.doMeasureBackground:
1698 self.log.info("Measuring background level.")
1699 self.measureBackground(ccdExposure, self.config.qa)
1701 if self.config.qa is not None and self.config.qa.saveStats is True:
1702 for amp in ccd:
1703 ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
1704 qaStats = afwMath.makeStatistics(ampExposure.getImage(),
1705 afwMath.MEDIAN | afwMath.STDEVCLIP)
1706 self.metadata.set("ISR BACKGROUND {} MEDIAN".format(amp.getName()),
1707 qaStats.getValue(afwMath.MEDIAN))
1708 self.metadata.set("ISR BACKGROUND {} STDEV".format(amp.getName()),
1709 qaStats.getValue(afwMath.STDEVCLIP))
1710 self.log.debug(" Background stats for amplifer %s: %f +/- %f",
1711 amp.getName(), qaStats.getValue(afwMath.MEDIAN),
1712 qaStats.getValue(afwMath.STDEVCLIP))
1714 self.debugView(ccdExposure, "postISRCCD")
1716 return pipeBase.Struct(
1717 exposure=ccdExposure,
1718 ossThumb=ossThumb,
1719 flattenedThumb=flattenedThumb,
1721 preInterpExposure=preInterpExp,
1722 outputExposure=ccdExposure,
1723 outputOssThumbnail=ossThumb,
1724 outputFlattenedThumbnail=flattenedThumb,
1725 )
1727 @pipeBase.timeMethod
1728 def runDataRef(self, sensorRef):
1729 """Perform instrument signature removal on a ButlerDataRef of a Sensor.
1731 This method contains the `CmdLineTask` interface to the ISR
1732 processing. All IO is handled here, freeing the `run()` method
1733 to manage only pixel-level calculations. The steps performed
1734 are:
1735 - Read in necessary detrending/isr/calibration data.
1736 - Process raw exposure in `run()`.
1737 - Persist the ISR-corrected exposure as "postISRCCD" if
1738 config.doWrite=True.
1740 Parameters
1741 ----------
1742 sensorRef : `daf.persistence.butlerSubset.ButlerDataRef`
1743 DataRef of the detector data to be processed
1745 Returns
1746 -------
1747 result : `lsst.pipe.base.Struct`
1748 Result struct with component:
1749 - ``exposure`` : `afw.image.Exposure`
1750 The fully ISR corrected exposure.
1752 Raises
1753 ------
1754 RuntimeError
1755 Raised if a configuration option is set to True, but the
1756 required calibration data does not exist.
1758 """
1759 self.log.info("Performing ISR on sensor %s.", sensorRef.dataId)
1761 ccdExposure = sensorRef.get(self.config.datasetType)
1763 camera = sensorRef.get("camera")
1764 isrData = self.readIsrData(sensorRef, ccdExposure)
1766 result = self.run(ccdExposure, camera=camera, **isrData.getDict())
1768 if self.config.doWrite:
1769 sensorRef.put(result.exposure, "postISRCCD")
1770 if result.preInterpExposure is not None:
1771 sensorRef.put(result.preInterpExposure, "postISRCCD_uninterpolated")
1772 if result.ossThumb is not None:
1773 isrQa.writeThumbnail(sensorRef, result.ossThumb, "ossThumb")
1774 if result.flattenedThumb is not None:
1775 isrQa.writeThumbnail(sensorRef, result.flattenedThumb, "flattenedThumb")
1777 return result
1779 def getIsrExposure(self, dataRef, datasetType, dateObs=None, immediate=True):
1780 """Retrieve a calibration dataset for removing instrument signature.
1782 Parameters
1783 ----------
1785 dataRef : `daf.persistence.butlerSubset.ButlerDataRef`
1786 DataRef of the detector data to find calibration datasets
1787 for.
1788 datasetType : `str`
1789 Type of dataset to retrieve (e.g. 'bias', 'flat', etc).
1790 dateObs : `str`, optional
1791 Date of the observation. Used to correct butler failures
1792 when using fallback filters.
1793 immediate : `Bool`
1794 If True, disable butler proxies to enable error handling
1795 within this routine.
1797 Returns
1798 -------
1799 exposure : `lsst.afw.image.Exposure`
1800 Requested calibration frame.
1802 Raises
1803 ------
1804 RuntimeError
1805 Raised if no matching calibration frame can be found.
1806 """
1807 try:
1808 exp = dataRef.get(datasetType, immediate=immediate)
1809 except Exception as exc1:
1810 if not self.config.fallbackFilterName:
1811 raise RuntimeError("Unable to retrieve %s for %s: %s." % (datasetType, dataRef.dataId, exc1))
1812 try:
1813 if self.config.useFallbackDate and dateObs:
1814 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName,
1815 dateObs=dateObs, immediate=immediate)
1816 else:
1817 exp = dataRef.get(datasetType, filter=self.config.fallbackFilterName, immediate=immediate)
1818 except Exception as exc2:
1819 raise RuntimeError("Unable to retrieve %s for %s, even with fallback filter %s: %s AND %s." %
1820 (datasetType, dataRef.dataId, self.config.fallbackFilterName, exc1, exc2))
1821 self.log.warning("Using fallback calibration from filter %s.", self.config.fallbackFilterName)
1823 if self.config.doAssembleIsrExposures:
1824 exp = self.assembleCcd.assembleCcd(exp)
1825 return exp
1827 def ensureExposure(self, inputExp, camera, detectorNum):
1828 """Ensure that the data returned by Butler is a fully constructed exposure.
1830 ISR requires exposure-level image data for historical reasons, so if we did
1831 not recieve that from Butler, construct it from what we have, modifying the
1832 input in place.
1834 Parameters
1835 ----------
1836 inputExp : `lsst.afw.image.Exposure`, `lsst.afw.image.DecoratedImageU`, or
1837 `lsst.afw.image.ImageF`
1838 The input data structure obtained from Butler.
1839 camera : `lsst.afw.cameraGeom.camera`
1840 The camera associated with the image. Used to find the appropriate
1841 detector.
1842 detectorNum : `int`
1843 The detector this exposure should match.
1845 Returns
1846 -------
1847 inputExp : `lsst.afw.image.Exposure`
1848 The re-constructed exposure, with appropriate detector parameters.
1850 Raises
1851 ------
1852 TypeError
1853 Raised if the input data cannot be used to construct an exposure.
1854 """
1855 if isinstance(inputExp, afwImage.DecoratedImageU):
1856 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1857 elif isinstance(inputExp, afwImage.ImageF):
1858 inputExp = afwImage.makeExposure(afwImage.makeMaskedImage(inputExp))
1859 elif isinstance(inputExp, afwImage.MaskedImageF):
1860 inputExp = afwImage.makeExposure(inputExp)
1861 elif isinstance(inputExp, afwImage.Exposure):
1862 pass
1863 elif inputExp is None:
1864 # Assume this will be caught by the setup if it is a problem.
1865 return inputExp
1866 else:
1867 raise TypeError("Input Exposure is not known type in isrTask.ensureExposure: %s." %
1868 (type(inputExp), ))
1870 if inputExp.getDetector() is None:
1871 inputExp.setDetector(camera[detectorNum])
1873 return inputExp
1875 def convertIntToFloat(self, exposure):
1876 """Convert exposure image from uint16 to float.
1878 If the exposure does not need to be converted, the input is
1879 immediately returned. For exposures that are converted to use
1880 floating point pixels, the variance is set to unity and the
1881 mask to zero.
1883 Parameters
1884 ----------
1885 exposure : `lsst.afw.image.Exposure`
1886 The raw exposure to be converted.
1888 Returns
1889 -------
1890 newexposure : `lsst.afw.image.Exposure`
1891 The input ``exposure``, converted to floating point pixels.
1893 Raises
1894 ------
1895 RuntimeError
1896 Raised if the exposure type cannot be converted to float.
1898 """
1899 if isinstance(exposure, afwImage.ExposureF):
1900 # Nothing to be done
1901 self.log.debug("Exposure already of type float.")
1902 return exposure
1903 if not hasattr(exposure, "convertF"):
1904 raise RuntimeError("Unable to convert exposure (%s) to float." % type(exposure))
1906 newexposure = exposure.convertF()
1907 newexposure.variance[:] = 1
1908 newexposure.mask[:] = 0x0
1910 return newexposure
1912 def maskAmplifier(self, ccdExposure, amp, defects):
1913 """Identify bad amplifiers, saturated and suspect pixels.
1915 Parameters
1916 ----------
1917 ccdExposure : `lsst.afw.image.Exposure`
1918 Input exposure to be masked.
1919 amp : `lsst.afw.table.AmpInfoCatalog`
1920 Catalog of parameters defining the amplifier on this
1921 exposure to mask.
1922 defects : `lsst.ip.isr.Defects`
1923 List of defects. Used to determine if the entire
1924 amplifier is bad.
1926 Returns
1927 -------
1928 badAmp : `Bool`
1929 If this is true, the entire amplifier area is covered by
1930 defects and unusable.
1932 """
1933 maskedImage = ccdExposure.getMaskedImage()
1935 badAmp = False
1937 # Check if entire amp region is defined as a defect (need to use amp.getBBox() for correct
1938 # comparison with current defects definition.
1939 if defects is not None:
1940 badAmp = bool(sum([v.getBBox().contains(amp.getBBox()) for v in defects]))
1942 # In the case of a bad amp, we will set mask to "BAD" (here use amp.getRawBBox() for correct
1943 # association with pixels in current ccdExposure).
1944 if badAmp:
1945 dataView = afwImage.MaskedImageF(maskedImage, amp.getRawBBox(),
1946 afwImage.PARENT)
1947 maskView = dataView.getMask()
1948 maskView |= maskView.getPlaneBitMask("BAD")
1949 del maskView
1950 return badAmp
1952 # Mask remaining defects after assembleCcd() to allow for defects that cross amplifier boundaries.
1953 # Saturation and suspect pixels can be masked now, though.
1954 limits = dict()
1955 if self.config.doSaturation and not badAmp:
1956 limits.update({self.config.saturatedMaskName: amp.getSaturation()})
1957 if self.config.doSuspect and not badAmp:
1958 limits.update({self.config.suspectMaskName: amp.getSuspectLevel()})
1959 if math.isfinite(self.config.saturation):
1960 limits.update({self.config.saturatedMaskName: self.config.saturation})
1962 for maskName, maskThreshold in limits.items():
1963 if not math.isnan(maskThreshold):
1964 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
1965 isrFunctions.makeThresholdMask(
1966 maskedImage=dataView,
1967 threshold=maskThreshold,
1968 growFootprints=0,
1969 maskName=maskName
1970 )
1972 # Determine if we've fully masked this amplifier with SUSPECT and SAT pixels.
1973 maskView = afwImage.Mask(maskedImage.getMask(), amp.getRawDataBBox(),
1974 afwImage.PARENT)
1975 maskVal = maskView.getPlaneBitMask([self.config.saturatedMaskName,
1976 self.config.suspectMaskName])
1977 if numpy.all(maskView.getArray() & maskVal > 0):
1978 badAmp = True
1979 maskView |= maskView.getPlaneBitMask("BAD")
1981 return badAmp
1983 def overscanCorrection(self, ccdExposure, amp):
1984 """Apply overscan correction in place.
1986 This method does initial pixel rejection of the overscan
1987 region. The overscan can also be optionally segmented to
1988 allow for discontinuous overscan responses to be fit
1989 separately. The actual overscan subtraction is performed by
1990 the `lsst.ip.isr.isrFunctions.overscanCorrection` function,
1991 which is called here after the amplifier is preprocessed.
1993 Parameters
1994 ----------
1995 ccdExposure : `lsst.afw.image.Exposure`
1996 Exposure to have overscan correction performed.
1997 amp : `lsst.afw.cameraGeom.Amplifer`
1998 The amplifier to consider while correcting the overscan.
2000 Returns
2001 -------
2002 overscanResults : `lsst.pipe.base.Struct`
2003 Result struct with components:
2004 - ``imageFit`` : scalar or `lsst.afw.image.Image`
2005 Value or fit subtracted from the amplifier image data.
2006 - ``overscanFit`` : scalar or `lsst.afw.image.Image`
2007 Value or fit subtracted from the overscan image data.
2008 - ``overscanImage`` : `lsst.afw.image.Image`
2009 Image of the overscan region with the overscan
2010 correction applied. This quantity is used to estimate
2011 the amplifier read noise empirically.
2013 Raises
2014 ------
2015 RuntimeError
2016 Raised if the ``amp`` does not contain raw pixel information.
2018 See Also
2019 --------
2020 lsst.ip.isr.isrFunctions.overscanCorrection
2021 """
2022 if amp.getRawHorizontalOverscanBBox().isEmpty():
2023 self.log.info("ISR_OSCAN: No overscan region. Not performing overscan correction.")
2024 return None
2026 statControl = afwMath.StatisticsControl()
2027 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2029 # Determine the bounding boxes
2030 dataBBox = amp.getRawDataBBox()
2031 oscanBBox = amp.getRawHorizontalOverscanBBox()
2032 dx0 = 0
2033 dx1 = 0
2035 prescanBBox = amp.getRawPrescanBBox()
2036 if (oscanBBox.getBeginX() > prescanBBox.getBeginX()): # amp is at the right
2037 dx0 += self.config.overscanNumLeadingColumnsToSkip
2038 dx1 -= self.config.overscanNumTrailingColumnsToSkip
2039 else:
2040 dx0 += self.config.overscanNumTrailingColumnsToSkip
2041 dx1 -= self.config.overscanNumLeadingColumnsToSkip
2043 # Determine if we need to work on subregions of the amplifier and overscan.
2044 imageBBoxes = []
2045 overscanBBoxes = []
2047 if ((self.config.overscanBiasJump
2048 and self.config.overscanBiasJumpLocation)
2049 and (ccdExposure.getMetadata().exists(self.config.overscanBiasJumpKeyword)
2050 and ccdExposure.getMetadata().getScalar(self.config.overscanBiasJumpKeyword) in
2051 self.config.overscanBiasJumpDevices)):
2052 if amp.getReadoutCorner() in (ReadoutCorner.LL, ReadoutCorner.LR):
2053 yLower = self.config.overscanBiasJumpLocation
2054 yUpper = dataBBox.getHeight() - yLower
2055 else:
2056 yUpper = self.config.overscanBiasJumpLocation
2057 yLower = dataBBox.getHeight() - yUpper
2059 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2060 lsst.geom.Extent2I(dataBBox.getWidth(), yLower)))
2061 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2062 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2063 yLower)))
2065 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin() + lsst.geom.Extent2I(0, yLower),
2066 lsst.geom.Extent2I(dataBBox.getWidth(), yUpper)))
2067 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, yLower),
2068 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2069 yUpper)))
2070 else:
2071 imageBBoxes.append(lsst.geom.Box2I(dataBBox.getBegin(),
2072 lsst.geom.Extent2I(dataBBox.getWidth(), dataBBox.getHeight())))
2073 overscanBBoxes.append(lsst.geom.Box2I(oscanBBox.getBegin() + lsst.geom.Extent2I(dx0, 0),
2074 lsst.geom.Extent2I(oscanBBox.getWidth() - dx0 + dx1,
2075 oscanBBox.getHeight())))
2077 # Perform overscan correction on subregions, ensuring saturated pixels are masked.
2078 for imageBBox, overscanBBox in zip(imageBBoxes, overscanBBoxes):
2079 ampImage = ccdExposure.maskedImage[imageBBox]
2080 overscanImage = ccdExposure.maskedImage[overscanBBox]
2082 overscanArray = overscanImage.image.array
2083 median = numpy.ma.median(numpy.ma.masked_where(overscanImage.mask.array, overscanArray))
2084 bad = numpy.where(numpy.abs(overscanArray - median) > self.config.overscanMaxDev)
2085 overscanImage.mask.array[bad] = overscanImage.mask.getPlaneBitMask("SAT")
2087 statControl = afwMath.StatisticsControl()
2088 statControl.setAndMask(ccdExposure.mask.getPlaneBitMask("SAT"))
2090 overscanResults = self.overscan.run(ampImage.getImage(), overscanImage, amp)
2092 # Measure average overscan levels and record them in the metadata.
2093 levelStat = afwMath.MEDIAN
2094 sigmaStat = afwMath.STDEVCLIP
2096 sctrl = afwMath.StatisticsControl(self.config.qa.flatness.clipSigma,
2097 self.config.qa.flatness.nIter)
2098 metadata = ccdExposure.getMetadata()
2099 ampNum = amp.getName()
2100 # if self.config.overscanFitType in ("MEDIAN", "MEAN", "MEANCLIP"):
2101 if isinstance(overscanResults.overscanFit, float):
2102 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, overscanResults.overscanFit)
2103 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, 0.0)
2104 else:
2105 stats = afwMath.makeStatistics(overscanResults.overscanFit, levelStat | sigmaStat, sctrl)
2106 metadata.set("ISR_OSCAN_LEVEL%s" % ampNum, stats.getValue(levelStat))
2107 metadata.set("ISR_OSCAN_SIGMA%s" % ampNum, stats.getValue(sigmaStat))
2109 return overscanResults
2111 def updateVariance(self, ampExposure, amp, overscanImage=None, ptcDataset=None):
2112 """Set the variance plane using the gain and read noise
2114 The read noise is calculated from the ``overscanImage`` if the
2115 ``doEmpiricalReadNoise`` option is set in the configuration; otherwise
2116 the value from the amplifier data is used.
2118 Parameters
2119 ----------
2120 ampExposure : `lsst.afw.image.Exposure`
2121 Exposure to process.
2122 amp : `lsst.afw.table.AmpInfoRecord` or `FakeAmp`
2123 Amplifier detector data.
2124 overscanImage : `lsst.afw.image.MaskedImage`, optional.
2125 Image of overscan, required only for empirical read noise.
2126 ptcDataset : `lsst.ip.isr.PhotonTransferCurveDataset`, optional
2127 PTC dataset containing the gains and read noise.
2130 Raises
2131 ------
2132 RuntimeError
2133 Raised if either ``usePtcGains`` of ``usePtcReadNoise``
2134 are ``True``, but ptcDataset is not provided.
2136 Raised if ```doEmpiricalReadNoise`` is ``True`` but
2137 ``overscanImage`` is ``None``.
2139 See also
2140 --------
2141 lsst.ip.isr.isrFunctions.updateVariance
2142 """
2143 maskPlanes = [self.config.saturatedMaskName, self.config.suspectMaskName]
2144 if self.config.usePtcGains:
2145 if ptcDataset is None:
2146 raise RuntimeError("No ptcDataset provided to use PTC gains.")
2147 else:
2148 gain = ptcDataset.gain[amp.getName()]
2149 self.log.info("Using gain from Photon Transfer Curve.")
2150 else:
2151 gain = amp.getGain()
2153 if math.isnan(gain):
2154 gain = 1.0
2155 self.log.warning("Gain set to NAN! Updating to 1.0 to generate Poisson variance.")
2156 elif gain <= 0:
2157 patchedGain = 1.0
2158 self.log.warning("Gain for amp %s == %g <= 0; setting to %f.",
2159 amp.getName(), gain, patchedGain)
2160 gain = patchedGain
2162 if self.config.doEmpiricalReadNoise and overscanImage is None:
2163 raise RuntimeError("Overscan is none for EmpiricalReadNoise.")
2165 if self.config.doEmpiricalReadNoise and overscanImage is not None:
2166 stats = afwMath.StatisticsControl()
2167 stats.setAndMask(overscanImage.mask.getPlaneBitMask(maskPlanes))
2168 readNoise = afwMath.makeStatistics(overscanImage, afwMath.STDEVCLIP, stats).getValue()
2169 self.log.info("Calculated empirical read noise for amp %s: %f.",
2170 amp.getName(), readNoise)
2171 elif self.config.usePtcReadNoise:
2172 if ptcDataset is None:
2173 raise RuntimeError("No ptcDataset provided to use PTC readnoise.")
2174 else:
2175 readNoise = ptcDataset.noise[amp.getName()]
2176 self.log.info("Using read noise from Photon Transfer Curve.")
2177 else:
2178 readNoise = amp.getReadNoise()
2180 isrFunctions.updateVariance(
2181 maskedImage=ampExposure.getMaskedImage(),
2182 gain=gain,
2183 readNoise=readNoise,
2184 )
2186 def darkCorrection(self, exposure, darkExposure, invert=False):
2187 """Apply dark correction in place.
2189 Parameters
2190 ----------
2191 exposure : `lsst.afw.image.Exposure`
2192 Exposure to process.
2193 darkExposure : `lsst.afw.image.Exposure`
2194 Dark exposure of the same size as ``exposure``.
2195 invert : `Bool`, optional
2196 If True, re-add the dark to an already corrected image.
2198 Raises
2199 ------
2200 RuntimeError
2201 Raised if either ``exposure`` or ``darkExposure`` do not
2202 have their dark time defined.
2204 See Also
2205 --------
2206 lsst.ip.isr.isrFunctions.darkCorrection
2207 """
2208 expScale = exposure.getInfo().getVisitInfo().getDarkTime()
2209 if math.isnan(expScale):
2210 raise RuntimeError("Exposure darktime is NAN.")
2211 if darkExposure.getInfo().getVisitInfo() is not None \
2212 and not math.isnan(darkExposure.getInfo().getVisitInfo().getDarkTime()):
2213 darkScale = darkExposure.getInfo().getVisitInfo().getDarkTime()
2214 else:
2215 # DM-17444: darkExposure.getInfo.getVisitInfo() is None
2216 # so getDarkTime() does not exist.
2217 self.log.warning("darkExposure.getInfo().getVisitInfo() does not exist. Using darkScale = 1.0.")
2218 darkScale = 1.0
2220 isrFunctions.darkCorrection(
2221 maskedImage=exposure.getMaskedImage(),
2222 darkMaskedImage=darkExposure.getMaskedImage(),
2223 expScale=expScale,
2224 darkScale=darkScale,
2225 invert=invert,
2226 trimToFit=self.config.doTrimToMatchCalib
2227 )
2229 def doLinearize(self, detector):
2230 """Check if linearization is needed for the detector cameraGeom.
2232 Checks config.doLinearize and the linearity type of the first
2233 amplifier.
2235 Parameters
2236 ----------
2237 detector : `lsst.afw.cameraGeom.Detector`
2238 Detector to get linearity type from.
2240 Returns
2241 -------
2242 doLinearize : `Bool`
2243 If True, linearization should be performed.
2244 """
2245 return self.config.doLinearize and \
2246 detector.getAmplifiers()[0].getLinearityType() != NullLinearityType
2248 def flatCorrection(self, exposure, flatExposure, invert=False):
2249 """Apply flat correction in place.
2251 Parameters
2252 ----------
2253 exposure : `lsst.afw.image.Exposure`
2254 Exposure to process.
2255 flatExposure : `lsst.afw.image.Exposure`
2256 Flat exposure of the same size as ``exposure``.
2257 invert : `Bool`, optional
2258 If True, unflatten an already flattened image.
2260 See Also
2261 --------
2262 lsst.ip.isr.isrFunctions.flatCorrection
2263 """
2264 isrFunctions.flatCorrection(
2265 maskedImage=exposure.getMaskedImage(),
2266 flatMaskedImage=flatExposure.getMaskedImage(),
2267 scalingType=self.config.flatScalingType,
2268 userScale=self.config.flatUserScale,
2269 invert=invert,
2270 trimToFit=self.config.doTrimToMatchCalib
2271 )
2273 def saturationDetection(self, exposure, amp):
2274 """Detect saturated pixels and mask them using mask plane config.saturatedMaskName, in place.
2276 Parameters
2277 ----------
2278 exposure : `lsst.afw.image.Exposure`
2279 Exposure to process. Only the amplifier DataSec is processed.
2280 amp : `lsst.afw.table.AmpInfoCatalog`
2281 Amplifier detector data.
2283 See Also
2284 --------
2285 lsst.ip.isr.isrFunctions.makeThresholdMask
2286 """
2287 if not math.isnan(amp.getSaturation()):
2288 maskedImage = exposure.getMaskedImage()
2289 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2290 isrFunctions.makeThresholdMask(
2291 maskedImage=dataView,
2292 threshold=amp.getSaturation(),
2293 growFootprints=0,
2294 maskName=self.config.saturatedMaskName,
2295 )
2297 def saturationInterpolation(self, exposure):
2298 """Interpolate over saturated pixels, in place.
2300 This method should be called after `saturationDetection`, to
2301 ensure that the saturated pixels have been identified in the
2302 SAT mask. It should also be called after `assembleCcd`, since
2303 saturated regions may cross amplifier boundaries.
2305 Parameters
2306 ----------
2307 exposure : `lsst.afw.image.Exposure`
2308 Exposure to process.
2310 See Also
2311 --------
2312 lsst.ip.isr.isrTask.saturationDetection
2313 lsst.ip.isr.isrFunctions.interpolateFromMask
2314 """
2315 isrFunctions.interpolateFromMask(
2316 maskedImage=exposure.getMaskedImage(),
2317 fwhm=self.config.fwhm,
2318 growSaturatedFootprints=self.config.growSaturationFootprintSize,
2319 maskNameList=list(self.config.saturatedMaskName),
2320 )
2322 def suspectDetection(self, exposure, amp):
2323 """Detect suspect pixels and mask them using mask plane config.suspectMaskName, in place.
2325 Parameters
2326 ----------
2327 exposure : `lsst.afw.image.Exposure`
2328 Exposure to process. Only the amplifier DataSec is processed.
2329 amp : `lsst.afw.table.AmpInfoCatalog`
2330 Amplifier detector data.
2332 See Also
2333 --------
2334 lsst.ip.isr.isrFunctions.makeThresholdMask
2336 Notes
2337 -----
2338 Suspect pixels are pixels whose value is greater than amp.getSuspectLevel().
2339 This is intended to indicate pixels that may be affected by unknown systematics;
2340 for example if non-linearity corrections above a certain level are unstable
2341 then that would be a useful value for suspectLevel. A value of `nan` indicates
2342 that no such level exists and no pixels are to be masked as suspicious.
2343 """
2344 suspectLevel = amp.getSuspectLevel()
2345 if math.isnan(suspectLevel):
2346 return
2348 maskedImage = exposure.getMaskedImage()
2349 dataView = maskedImage.Factory(maskedImage, amp.getRawBBox())
2350 isrFunctions.makeThresholdMask(
2351 maskedImage=dataView,
2352 threshold=suspectLevel,
2353 growFootprints=0,
2354 maskName=self.config.suspectMaskName,
2355 )
2357 def maskDefect(self, exposure, defectBaseList):
2358 """Mask defects using mask plane "BAD", in place.
2360 Parameters
2361 ----------
2362 exposure : `lsst.afw.image.Exposure`
2363 Exposure to process.
2364 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2365 `lsst.afw.image.DefectBase`.
2366 List of defects to mask.
2368 Notes
2369 -----
2370 Call this after CCD assembly, since defects may cross amplifier boundaries.
2371 """
2372 maskedImage = exposure.getMaskedImage()
2373 if not isinstance(defectBaseList, Defects):
2374 # Promotes DefectBase to Defect
2375 defectList = Defects(defectBaseList)
2376 else:
2377 defectList = defectBaseList
2378 defectList.maskPixels(maskedImage, maskName="BAD")
2380 def maskEdges(self, exposure, numEdgePixels=0, maskPlane="SUSPECT", level='DETECTOR'):
2381 """Mask edge pixels with applicable mask plane.
2383 Parameters
2384 ----------
2385 exposure : `lsst.afw.image.Exposure`
2386 Exposure to process.
2387 numEdgePixels : `int`, optional
2388 Number of edge pixels to mask.
2389 maskPlane : `str`, optional
2390 Mask plane name to use.
2391 level : `str`, optional
2392 Level at which to mask edges.
2393 """
2394 maskedImage = exposure.getMaskedImage()
2395 maskBitMask = maskedImage.getMask().getPlaneBitMask(maskPlane)
2397 if numEdgePixels > 0:
2398 if level == 'DETECTOR':
2399 boxes = [maskedImage.getBBox()]
2400 elif level == 'AMP':
2401 boxes = [amp.getBBox() for amp in exposure.getDetector()]
2403 for box in boxes:
2404 # This makes a bbox numEdgeSuspect pixels smaller than the image on each side
2405 subImage = maskedImage[box]
2406 box.grow(-numEdgePixels)
2407 # Mask pixels outside box
2408 SourceDetectionTask.setEdgeBits(
2409 subImage,
2410 box,
2411 maskBitMask)
2413 def maskAndInterpolateDefects(self, exposure, defectBaseList):
2414 """Mask and interpolate defects using mask plane "BAD", in place.
2416 Parameters
2417 ----------
2418 exposure : `lsst.afw.image.Exposure`
2419 Exposure to process.
2420 defectBaseList : `lsst.ip.isr.Defects` or `list` of
2421 `lsst.afw.image.DefectBase`.
2422 List of defects to mask and interpolate.
2424 See Also
2425 --------
2426 lsst.ip.isr.isrTask.maskDefect
2427 """
2428 self.maskDefect(exposure, defectBaseList)
2429 self.maskEdges(exposure, numEdgePixels=self.config.numEdgeSuspect,
2430 maskPlane="SUSPECT", level=self.config.edgeMaskLevel)
2431 isrFunctions.interpolateFromMask(
2432 maskedImage=exposure.getMaskedImage(),
2433 fwhm=self.config.fwhm,
2434 growSaturatedFootprints=0,
2435 maskNameList=["BAD"],
2436 )
2438 def maskNan(self, exposure):
2439 """Mask NaNs using mask plane "UNMASKEDNAN", in place.
2441 Parameters
2442 ----------
2443 exposure : `lsst.afw.image.Exposure`
2444 Exposure to process.
2446 Notes
2447 -----
2448 We mask over all non-finite values (NaN, inf), including those
2449 that are masked with other bits (because those may or may not be
2450 interpolated over later, and we want to remove all NaN/infs).
2451 Despite this behaviour, the "UNMASKEDNAN" mask plane is used to
2452 preserve the historical name.
2453 """
2454 maskedImage = exposure.getMaskedImage()
2456 # Find and mask NaNs
2457 maskedImage.getMask().addMaskPlane("UNMASKEDNAN")
2458 maskVal = maskedImage.getMask().getPlaneBitMask("UNMASKEDNAN")
2459 numNans = maskNans(maskedImage, maskVal)
2460 self.metadata.set("NUMNANS", numNans)
2461 if numNans > 0:
2462 self.log.warning("There were %d unmasked NaNs.", numNans)
2464 def maskAndInterpolateNan(self, exposure):
2465 """"Mask and interpolate NaN/infs using mask plane "UNMASKEDNAN",
2466 in place.
2468 Parameters
2469 ----------
2470 exposure : `lsst.afw.image.Exposure`
2471 Exposure to process.
2473 See Also
2474 --------
2475 lsst.ip.isr.isrTask.maskNan
2476 """
2477 self.maskNan(exposure)
2478 isrFunctions.interpolateFromMask(
2479 maskedImage=exposure.getMaskedImage(),
2480 fwhm=self.config.fwhm,
2481 growSaturatedFootprints=0,
2482 maskNameList=["UNMASKEDNAN"],
2483 )
2485 def measureBackground(self, exposure, IsrQaConfig=None):
2486 """Measure the image background in subgrids, for quality control purposes.
2488 Parameters
2489 ----------
2490 exposure : `lsst.afw.image.Exposure`
2491 Exposure to process.
2492 IsrQaConfig : `lsst.ip.isr.isrQa.IsrQaConfig`
2493 Configuration object containing parameters on which background
2494 statistics and subgrids to use.
2495 """
2496 if IsrQaConfig is not None:
2497 statsControl = afwMath.StatisticsControl(IsrQaConfig.flatness.clipSigma,
2498 IsrQaConfig.flatness.nIter)
2499 maskVal = exposure.getMaskedImage().getMask().getPlaneBitMask(["BAD", "SAT", "DETECTED"])
2500 statsControl.setAndMask(maskVal)
2501 maskedImage = exposure.getMaskedImage()
2502 stats = afwMath.makeStatistics(maskedImage, afwMath.MEDIAN | afwMath.STDEVCLIP, statsControl)
2503 skyLevel = stats.getValue(afwMath.MEDIAN)
2504 skySigma = stats.getValue(afwMath.STDEVCLIP)
2505 self.log.info("Flattened sky level: %f +/- %f.", skyLevel, skySigma)
2506 metadata = exposure.getMetadata()
2507 metadata.set('SKYLEVEL', skyLevel)
2508 metadata.set('SKYSIGMA', skySigma)
2510 # calcluating flatlevel over the subgrids
2511 stat = afwMath.MEANCLIP if IsrQaConfig.flatness.doClip else afwMath.MEAN
2512 meshXHalf = int(IsrQaConfig.flatness.meshX/2.)
2513 meshYHalf = int(IsrQaConfig.flatness.meshY/2.)
2514 nX = int((exposure.getWidth() + meshXHalf) / IsrQaConfig.flatness.meshX)
2515 nY = int((exposure.getHeight() + meshYHalf) / IsrQaConfig.flatness.meshY)
2516 skyLevels = numpy.zeros((nX, nY))
2518 for j in range(nY):
2519 yc = meshYHalf + j * IsrQaConfig.flatness.meshY
2520 for i in range(nX):
2521 xc = meshXHalf + i * IsrQaConfig.flatness.meshX
2523 xLLC = xc - meshXHalf
2524 yLLC = yc - meshYHalf
2525 xURC = xc + meshXHalf - 1
2526 yURC = yc + meshYHalf - 1
2528 bbox = lsst.geom.Box2I(lsst.geom.Point2I(xLLC, yLLC), lsst.geom.Point2I(xURC, yURC))
2529 miMesh = maskedImage.Factory(exposure.getMaskedImage(), bbox, afwImage.LOCAL)
2531 skyLevels[i, j] = afwMath.makeStatistics(miMesh, stat, statsControl).getValue()
2533 good = numpy.where(numpy.isfinite(skyLevels))
2534 skyMedian = numpy.median(skyLevels[good])
2535 flatness = (skyLevels[good] - skyMedian) / skyMedian
2536 flatness_rms = numpy.std(flatness)
2537 flatness_pp = flatness.max() - flatness.min() if len(flatness) > 0 else numpy.nan
2539 self.log.info("Measuring sky levels in %dx%d grids: %f.", nX, nY, skyMedian)
2540 self.log.info("Sky flatness in %dx%d grids - pp: %f rms: %f.",
2541 nX, nY, flatness_pp, flatness_rms)
2543 metadata.set('FLATNESS_PP', float(flatness_pp))
2544 metadata.set('FLATNESS_RMS', float(flatness_rms))
2545 metadata.set('FLATNESS_NGRIDS', '%dx%d' % (nX, nY))
2546 metadata.set('FLATNESS_MESHX', IsrQaConfig.flatness.meshX)
2547 metadata.set('FLATNESS_MESHY', IsrQaConfig.flatness.meshY)
2549 def roughZeroPoint(self, exposure):
2550 """Set an approximate magnitude zero point for the exposure.
2552 Parameters
2553 ----------
2554 exposure : `lsst.afw.image.Exposure`
2555 Exposure to process.
2556 """
2557 filterLabel = exposure.getFilterLabel()
2558 physicalFilter = isrFunctions.getPhysicalFilter(filterLabel, self.log)
2560 if physicalFilter in self.config.fluxMag0T1:
2561 fluxMag0 = self.config.fluxMag0T1[physicalFilter]
2562 else:
2563 self.log.warning("No rough magnitude zero point defined for filter %s.", physicalFilter)
2564 fluxMag0 = self.config.defaultFluxMag0T1
2566 expTime = exposure.getInfo().getVisitInfo().getExposureTime()
2567 if not expTime > 0: # handle NaN as well as <= 0
2568 self.log.warning("Non-positive exposure time; skipping rough zero point.")
2569 return
2571 self.log.info("Setting rough magnitude zero point for filter %s: %f",
2572 physicalFilter, 2.5*math.log10(fluxMag0*expTime))
2573 exposure.setPhotoCalib(afwImage.makePhotoCalibFromCalibZeroPoint(fluxMag0*expTime, 0.0))
2575 def setValidPolygonIntersect(self, ccdExposure, fpPolygon):
2576 """Set the valid polygon as the intersection of fpPolygon and the ccd corners.
2578 Parameters
2579 ----------
2580 ccdExposure : `lsst.afw.image.Exposure`
2581 Exposure to process.
2582 fpPolygon : `lsst.afw.geom.Polygon`
2583 Polygon in focal plane coordinates.
2584 """
2585 # Get ccd corners in focal plane coordinates
2586 ccd = ccdExposure.getDetector()
2587 fpCorners = ccd.getCorners(FOCAL_PLANE)
2588 ccdPolygon = Polygon(fpCorners)
2590 # Get intersection of ccd corners with fpPolygon
2591 intersect = ccdPolygon.intersectionSingle(fpPolygon)
2593 # Transform back to pixel positions and build new polygon
2594 ccdPoints = ccd.transform(intersect, FOCAL_PLANE, PIXELS)
2595 validPolygon = Polygon(ccdPoints)
2596 ccdExposure.getInfo().setValidPolygon(validPolygon)
2598 @contextmanager
2599 def flatContext(self, exp, flat, dark=None):
2600 """Context manager that applies and removes flats and darks,
2601 if the task is configured to apply them.
2603 Parameters
2604 ----------
2605 exp : `lsst.afw.image.Exposure`
2606 Exposure to process.
2607 flat : `lsst.afw.image.Exposure`
2608 Flat exposure the same size as ``exp``.
2609 dark : `lsst.afw.image.Exposure`, optional
2610 Dark exposure the same size as ``exp``.
2612 Yields
2613 ------
2614 exp : `lsst.afw.image.Exposure`
2615 The flat and dark corrected exposure.
2616 """
2617 if self.config.doDark and dark is not None:
2618 self.darkCorrection(exp, dark)
2619 if self.config.doFlat:
2620 self.flatCorrection(exp, flat)
2621 try:
2622 yield exp
2623 finally:
2624 if self.config.doFlat:
2625 self.flatCorrection(exp, flat, invert=True)
2626 if self.config.doDark and dark is not None:
2627 self.darkCorrection(exp, dark, invert=True)
2629 def debugView(self, exposure, stepname):
2630 """Utility function to examine ISR exposure at different stages.
2632 Parameters
2633 ----------
2634 exposure : `lsst.afw.image.Exposure`
2635 Exposure to view.
2636 stepname : `str`
2637 State of processing to view.
2638 """
2639 frame = getDebugFrame(self._display, stepname)
2640 if frame:
2641 display = getDisplay(frame)
2642 display.scale('asinh', 'zscale')
2643 display.mtv(exposure)
2644 prompt = "Press Enter to continue [c]... "
2645 while True:
2646 ans = input(prompt).lower()
2647 if ans in ("", "c",):
2648 break
2651class FakeAmp(object):
2652 """A Detector-like object that supports returning gain and saturation level
2654 This is used when the input exposure does not have a detector.
2656 Parameters
2657 ----------
2658 exposure : `lsst.afw.image.Exposure`
2659 Exposure to generate a fake amplifier for.
2660 config : `lsst.ip.isr.isrTaskConfig`
2661 Configuration to apply to the fake amplifier.
2662 """
2664 def __init__(self, exposure, config):
2665 self._bbox = exposure.getBBox(afwImage.LOCAL)
2666 self._RawHorizontalOverscanBBox = lsst.geom.Box2I()
2667 self._gain = config.gain
2668 self._readNoise = config.readNoise
2669 self._saturation = config.saturation
2671 def getBBox(self):
2672 return self._bbox
2674 def getRawBBox(self):
2675 return self._bbox
2677 def getRawHorizontalOverscanBBox(self):
2678 return self._RawHorizontalOverscanBBox
2680 def getGain(self):
2681 return self._gain
2683 def getReadNoise(self):
2684 return self._readNoise
2686 def getSaturation(self):
2687 return self._saturation
2689 def getSuspectLevel(self):
2690 return float("NaN")
2693class RunIsrConfig(pexConfig.Config):
2694 isr = pexConfig.ConfigurableField(target=IsrTask, doc="Instrument signature removal")
2697class RunIsrTask(pipeBase.CmdLineTask):
2698 """Task to wrap the default IsrTask to allow it to be retargeted.
2700 The standard IsrTask can be called directly from a command line
2701 program, but doing so removes the ability of the task to be
2702 retargeted. As most cameras override some set of the IsrTask
2703 methods, this would remove those data-specific methods in the
2704 output post-ISR images. This wrapping class fixes the issue,
2705 allowing identical post-ISR images to be generated by both the
2706 processCcd and isrTask code.
2707 """
2708 ConfigClass = RunIsrConfig
2709 _DefaultName = "runIsr"
2711 def __init__(self, *args, **kwargs):
2712 super().__init__(*args, **kwargs)
2713 self.makeSubtask("isr")
2715 def runDataRef(self, dataRef):
2716 """
2717 Parameters
2718 ----------
2719 dataRef : `lsst.daf.persistence.ButlerDataRef`
2720 data reference of the detector data to be processed
2722 Returns
2723 -------
2724 result : `pipeBase.Struct`
2725 Result struct with component:
2727 - exposure : `lsst.afw.image.Exposure`
2728 Post-ISR processed exposure.
2729 """
2730 return self.isr.runDataRef(dataRef)