Coverage for python/lsst/cp/verify/verifyDark.py: 10%
65 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-10 11:36 +0000
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-10 11:36 +0000
1# This file is part of cp_verify.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21import numpy as np
23from .verifyStats import CpVerifyStatsConfig, CpVerifyStatsTask, CpVerifyStatsConnections
26__all__ = ['CpVerifyDarkConfig', 'CpVerifyDarkTask']
29class CpVerifyDarkConfig(CpVerifyStatsConfig,
30 pipelineConnections=CpVerifyStatsConnections):
31 """Inherits from base CpVerifyStatsConfig.
32 """
34 def setDefaults(self):
35 super().setDefaults()
36 self.stageName = 'DARK'
37 self.imageStatKeywords = {'MEAN': 'MEAN', # noqa F841
38 'NOISE': 'STDEVCLIP', }
39 self.crImageStatKeywords = {'CR_NOISE': 'STDEV', } # noqa F841
40 self.metadataStatKeywords = {'RESIDUAL STDEV': 'AMP', } # noqa F841
43class CpVerifyDarkTask(CpVerifyStatsTask):
44 """Dark verification sub-class, implementing the verify method.
45 """
46 ConfigClass = CpVerifyDarkConfig
47 _DefaultName = 'cpVerifyDark'
49 def verify(self, exposure, statisticsDict):
50 """Verify that the measured statistics meet the verification criteria.
52 Parameters
53 ----------
54 exposure : `lsst.afw.image.Exposure`
55 The exposure the statistics are from.
56 statisticsDictionary : `dict` [`str`, `dict` [`str`, scalar]],
57 Dictionary of measured statistics. The inner dictionary
58 should have keys that are statistic names (`str`) with
59 values that are some sort of scalar (`int` or `float` are
60 the mostly likely types).
62 Returns
63 -------
64 outputStatistics : `dict` [`str`, `dict` [`str`, `bool`]]
65 A dictionary indexed by the amplifier name, containing
66 dictionaries of the verification criteria.
67 success : `bool`
68 A boolean indicating if all tests have passed.
69 """
70 detector = exposure.getDetector()
71 ampStats = statisticsDict['AMP']
72 metadataStats = statisticsDict['METADATA']
74 verifyStats = {}
75 success = True
76 for ampName, stats in ampStats.items():
77 verify = {}
79 # DMTN-101 Test 5.2: Mean is 0.0:
80 verify['MEAN'] = bool(np.abs(stats['MEAN']) < stats['NOISE'])
82 # DMTN-101 Test 5.3: Clipped mean matches readNoise. This
83 # test should use the nominal detector read noise. The
84 # f"RESIDUAL STDEV {ampName}" metadata entry contains the
85 # measured dispersion in the overscan-corrected overscan
86 # region, which should provide an estimate of the read
87 # noise. However, directly using this value will cause
88 # some fraction of verification runs to fail if the
89 # scatter in read noise values is comparable to the test
90 # threshold, as the overscan residual measured may be
91 # sampling from the low end tail of the distribution.
92 # This measurement is also likely to be smaller than that
93 # measured on the bulk of the image as the overscan
94 # correction should be an optimal fit to the overscan
95 # region, but not necessarily for the image region.
96 readNoise = detector[ampName].getReadNoise()
97 verify['NOISE'] = bool((stats['NOISE'] - readNoise)/readNoise <= 0.05)
99 # DMTN-101 Test 5.4: CR rejection matches clipped mean
100 verify['CR_NOISE'] = bool(np.abs(stats['NOISE'] - stats['CR_NOISE'])/stats['CR_NOISE'] <= 0.05)
102 # Confirm this hasn't triggered a raise condition.
103 if 'FORCE_FAILURE' in stats:
104 verify['PROCESSING'] = False
106 verify['SUCCESS'] = bool(np.all(list(verify.values())))
107 if verify['SUCCESS'] is False:
108 success = False
110 # After determining the verification status for this
111 # exposure, we can also check to see how well the read
112 # noise measured from the overscan residual matches the
113 # nominal value used above in Test 5.3. If these disagree
114 # consistently and significantly, then the assumptions
115 # used in that test may be incorrect, and the nominal read
116 # noise may need recalculation. Only perform this check
117 # if the metadataStats contain the required entry.
118 if 'RESIDUAL STDEV' in metadataStats and ampName in metadataStats['RESIDUAL STDEV']:
119 verify['READ_NOISE_CONSISTENT'] = True
120 overscanReadNoise = metadataStats['RESIDUAL STDEV'][ampName]
121 if overscanReadNoise:
122 if ((overscanReadNoise - readNoise)/readNoise > 0.05):
123 verify['READ_NOISE_CONSISTENT'] = False
125 verifyStats[ampName] = verify
127 return {'AMP': verifyStats}, bool(success)
129 def repackStats(self, statisticsDict, dimensions):
130 # docstring inherited
131 rows = {}
132 rowList = []
133 matrixRowList = None
135 if self.config.useIsrStatistics:
136 mjd = statisticsDict["ISR"]["MJD"]
137 else:
138 mjd = np.nan
140 rowBase = {
141 "instrument": dimensions["instrument"],
142 "exposure": dimensions["exposure"],
143 "detector": dimensions["detector"],
144 "mjd": mjd,
145 }
147 # AMP results:
148 for ampName, stats in statisticsDict["AMP"].items():
149 rows[ampName] = {}
150 rows[ampName].update(rowBase)
151 rows[ampName]["amplifier"] = ampName
152 for key, value in stats.items():
153 rows[ampName][f"{self.config.stageName}_{key}"] = value
155 # VERIFY results
156 for ampName, stats in statisticsDict["VERIFY"]["AMP"].items():
157 for key, value in stats.items():
158 rows[ampName][f"{self.config.stageName}_VERIFY_{key}"] = value
160 # METADATA results
161 for ampName, value in statisticsDict["METADATA"]["RESIDUAL STDEV"].items():
162 rows[ampName][f"{self.config.stageName}_READ_NOISE"] = value
164 # ISR results
165 if self.config.useIsrStatistics and "ISR" in statisticsDict:
166 for ampName, stats in statisticsDict["ISR"]["CALIBDIST"].items():
167 for level in self.config.expectedDistributionLevels:
168 key = f"LSST CALIB {self.config.stageName.upper()} {ampName} DISTRIBUTION {level}-PCT"
169 rows[ampName][f"{self.config.stageName}_DARK_DIST_{level}_PCT"] = stats[key]
171 # pack final list
172 for ampName, stats in rows.items():
173 rowList.append(stats)
175 return rowList, matrixRowList