Coverage for python / lsst / cp / verify / verifyDefects.py: 19%
99 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-04-15 00:24 +0000
1# This file is part of cp_verify.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21import numpy as np
22import scipy.stats
23import lsst.pipe.base.connectionTypes as cT
24from lsst.ip.isr.isrFunctions import countMaskedPixels
26from .verifyStats import (
27 CpVerifyStatsConfig,
28 CpVerifyStatsTask,
29 CpVerifyStatsConnections,
30)
33__all__ = ["CpVerifyDefectsConfig", "CpVerifyDefectsTask"]
36class CpVerifyDefectsConnections(
37 CpVerifyStatsConnections, dimensions={"instrument", "exposure", "detector"}
38):
39 inputExp = cT.Input(
40 name="icExp",
41 doc="Input exposure to calculate statistics for.",
42 storageClass="Exposure",
43 dimensions=["instrument", "exposure", "detector"],
44 )
45 uncorrectedExp = cT.Input(
46 name="uncorrectedExp",
47 doc="Uncorrected input exposure to calculate statistics for.",
48 storageClass="Exposure",
49 dimensions=["instrument", "exposure", "detector"],
50 )
51 camera = cT.PrerequisiteInput(
52 name="camera",
53 storageClass="Camera",
54 doc="Input camera.",
55 dimensions=["instrument", ],
56 isCalibration=True,
57 )
58 outputStats = cT.Output(
59 name="detectorStats",
60 doc="Output statistics from cp_verify.",
61 storageClass="StructuredDataDict",
62 dimensions=["instrument", "exposure", "detector"],
63 )
64 outputResults = cT.Output(
65 name="detectorResults",
66 doc="Output results from cp_verify.",
67 storageClass="ArrowAstropy",
68 dimensions=["instrument", "exposure", "detector"],
69 )
70 outputMatrix = cT.Output(
71 name="detectorMatrix",
72 doc="Output matrix results from cp_verify.",
73 storageClass="ArrowAstropy",
74 dimensions=["instrument", "exposure", "detector"],
75 )
78class CpVerifyDefectsConfig(
79 CpVerifyStatsConfig, pipelineConnections=CpVerifyDefectsConnections
80):
81 """Inherits from base CpVerifyStatsConfig."""
83 def setDefaults(self):
84 super().setDefaults()
85 self.stageName = 'DEFECTS'
87 self.maskNameList = ["BAD"] # noqa F821
89 self.imageStatKeywords = {
90 "DEFECT_PIXELS": "NMASKED", # noqa F821
91 "OUTLIERS": "NCLIPPED",
92 "MEDIAN": "MEDIAN",
93 "STDEV": "STDEVCLIP",
94 "MIN": "MIN",
95 "MAX": "MAX",
96 }
97 self.unmaskedImageStatKeywords = {
98 "UNMASKED_MIN": "MIN", # noqa F821
99 "UNMASKED_MAX": "MAX",
100 "UNMASKED_STDEV": "STDEVCLIP",
101 "UNMASKED_OUTLIERS": "NCLIPPED",
102 }
103 self.uncorrectedImageStatKeywords = {
104 "UNC_DEFECT_PIXELS": "NMASKED", # noqa F821
105 "UNC_OUTLIERS": "NCLIPPED",
106 "UNC_MEDIAN": "MEDIAN",
107 "UNC_STDEV": "STDEVCLIP",
108 "UNC_MIN": "MIN",
109 "UNC_MAX": "MAX",
110 }
111 # These config options need to have a key/value pair
112 # to run verification analysis, but the contents of
113 # that pair are not used.
114 self.catalogStatKeywords = {"empty": "dictionary"}
115 self.detectorStatKeywords = {"empty": "dictionary"}
118class CpVerifyDefectsTask(CpVerifyStatsTask):
119 """Defects verification sub-class, implementing the verify method.
121 This also applies additional image processing statistics.
122 """
124 ConfigClass = CpVerifyDefectsConfig
125 _DefaultName = "cpVerifyDefects"
127 def catalogStatistics(self, exposure, catalog, uncorrectedCatalog, statControl):
128 """Measure the catalog statistics.
130 Parameters
131 ----------
132 exposure : `lsst.afw.image.Exposure`
133 The exposure to measure.
134 catalog : `lsst.afw.table.Table`
135 The catalog to measure.
136 uncorrectedCatalog : `lsst.afw.table.Table`
137 The uncorrected catalog to measure.
138 statControl : `lsst.afw.math.StatisticsControl`
139 Statistics control object with parameters defined by
140 the config.
142 Returns
143 -------
144 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]]
145 A dictionary indexed by the amplifier name, containing
146 dictionaries of the statistics measured and their values.
148 Notes
149 -----
150 Number of detections test: running with defects would have fewer
151 detections
152 """
153 outputStatistics = {}
155 # Number of detections test
156 outputStatistics["NUM_OBJECTS_BEFORE"] = len(uncorrectedCatalog)
157 outputStatistics["NUM_OBJECTS_AFTER"] = len(catalog)
159 return outputStatistics
161 def detectorStatistics(self, statisticsDict, statControl, exposure=None, uncorrectedExposure=None):
162 """Measure the detector statistics.
164 Parameters
165 ----------
166 statisticsDict : `dict` [`str`, scalar]
167 Dictionary with detector tests.
168 statControl : `lsst.afw.math.StatControl`
169 Statistics control object with parameters defined by
170 the config.
171 exposure : `lsst.afw.image.Exposure`, optional
172 Exposure containing the ISR-processed data to measure.
173 uncorrectedExposure : `lsst.afw.image.Exposure`, optional
174 uncorrected esposure (no defects) containing the
175 ISR-processed data to measure.
177 Returns
178 -------
179 outputStatistics : `dict` [`str`, scalar]
180 A dictionary containing statistics measured and their values.
182 Notes
183 -----
184 Number of cosmic rays test: If there are defects in our data that
185 we didn't properly identify and cover, they might appear similar
186 to cosmic rays because they have sharp edges compared to the point
187 spread function (PSF). When we process the data, if these defects
188 aren't marked in our defect mask, the software might mistakenly think
189 they are cosmic rays and try to remove them. However, if we've already
190 included these defects in the defect mask, the software won't treat
191 them as cosmic rays, so we'll have fewer pixels that are falsely
192 identified and removed as cosmic rays when we compare two sets of
193 data reductions.
194 """
195 outputStatistics = {}
196 # Cosmic Rays test: Count number of cosmic rays before
197 # and after masking with defects
198 nCosmicsBefore = countMaskedPixels(uncorrectedExposure, ["CR"])
199 nCosmicsAfter = countMaskedPixels(exposure, ["CR"])
201 outputStatistics["NUM_COSMICS_BEFORE"] = nCosmicsBefore
202 outputStatistics["NUM_COSMICS_AFTER"] = nCosmicsAfter
204 return outputStatistics
206 def imageStatistics(self, exposure, uncorrectedExposure, statControl):
207 """Measure additional defect statistics.
209 This calls the parent class method first, then adds additional
210 measurements.
212 Parameters
213 ----------
214 exposure : `lsst.afw.image.Exposure`
215 Exposure containing the ISR processed data to measure.
216 statControl : `lsst.afw.math.StatControl`
217 Statistics control object with parameters defined by
218 the config.
220 Returns
221 -------
222 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]]
223 A dictionary indexed by the amplifier name, containing
224 dictionaries of the statistics measured and their values.
226 Notes
227 -----
228 Number of cosmic rays test: If there are defects in our data that
229 we didn't properly identify and cover, they might appear similar
230 to cosmic rays because they have sharp edges compared to the point
231 spread function (PSF). When we process the data, if these defects
232 aren't marked in our defect mask, the software might mistakenly think
233 they are cosmic rays and try to remove them. However, if we've already
234 included these defects in the defect mask, the software won't treat
235 them as cosmic rays, so we'll have fewer pixels that are falsely
236 identified and removed as cosmic rays when we compare two sets of
237 data reductions.
238 """
239 outputStatistics = super().imageStatistics(
240 exposure, uncorrectedExposure, statControl
241 )
243 # Is this a useful test? It saves having to do chi^2 fits,
244 # which are going to be biased by the bulk of points.
245 for amp in exposure.getDetector():
246 ampName = amp.getName()
247 ampExp = exposure.Factory(exposure, amp.getBBox())
249 normImage = ampExp.getImage()
250 normArray = normImage.getArray()
252 normArray -= outputStatistics[ampName]["MEDIAN"]
253 normArray /= outputStatistics[ampName]["STDEV"]
255 probability = scipy.stats.norm.pdf(normArray)
256 outliers = np.where(probability < 1.0 / probability.size, 1.0, 0.0)
257 outputStatistics[ampName]["STAT_OUTLIERS"] = int(np.sum(outliers))
259 # Get fraction of defects per amp
260 ampSize = amp.getBBox().height*amp.getBBox().width
261 outputStatistics[ampName]["FRAC"] = outputStatistics[ampName]["DEFECT_PIXELS"]/ampSize
263 return outputStatistics
265 def verify(self, exposure, statisticsDict):
266 """Verify that the measured statistics meet the verification criteria.
268 Parameters
269 ----------
270 exposure : `lsst.afw.image.Exposure`
271 The exposure the statistics are from.
272 statisticsDictionary : `dict` [`str`, `dict` [`str`, scalar]],
273 Dictionary of measured statistics. The inner dictionary
274 should have keys that are statistic names (`str`) with
275 values that are some sort of scalar (`int` or `float` are
276 the mostly likely types).
278 Returns
279 -------
280 outputStatistics : `dict` [`str`, `dict` [`str`, `bool`]]
281 A dictionary indexed by the amplifier name, containing
282 dictionaries of the verification criteria.
283 success : `bool`
284 A boolean indicating if all tests have passed.
285 """
286 # Amplifier statistics
287 ampStats = statisticsDict["AMP"]
288 verifyStats = {}
289 successAmp = True
290 for ampName, stats in ampStats.items():
291 verify = {}
293 # These are not defined in DMTN-101 yet.
294 verify["OUTLIERS"] = bool(stats["UNMASKED_OUTLIERS"] >= stats["OUTLIERS"])
295 verify["STDEV"] = bool(stats["UNMASKED_STDEV"] >= stats["STDEV"])
296 verify["MIN"] = bool(stats["UNMASKED_MIN"] <= stats["MIN"])
297 verify["MAX"] = bool(stats["UNMASKED_MAX"] >= stats["MAX"])
299 # This test is bad, and should be made not bad.
300 verify["PROB_TEST"] = bool(stats["STAT_OUTLIERS"] == stats["DEFECT_PIXELS"])
302 verify["SUCCESS"] = bool(np.all(list(verify.values())))
303 if verify["SUCCESS"] is False:
304 successAmp = False
306 verifyStats[ampName] = verify
308 success = successAmp
309 return {
310 "AMP": verifyStats
311 }, success
313 def repackStats(self, statisticsDict, dimensions):
314 # docstring inherited
315 rows = {}
316 rowList = []
317 matrixRowList = None
319 if self.config.useIsrStatistics:
320 mjd = statisticsDict["ISR"]["MJD"]
321 else:
322 mjd = np.nan
324 rowBase = {
325 "instrument": dimensions["instrument"],
326 "exposure": dimensions["exposure"], # ensure an exposure dimension for downstream.
327 "detector": dimensions["detector"],
328 "mjd": mjd,
329 }
331 # AMP results
332 for ampName, stats in statisticsDict["AMP"].items():
333 rows[ampName] = {}
334 rows[ampName].update(rowBase)
335 rows[ampName]["amplifier"] = ampName
336 for key, value in stats.items():
337 rows[ampName][f"{self.config.stageName}_{key}"] = value
339 # ISR results
340 nBadColumns = np.nan
341 if self.config.useIsrStatistics and "ISR" in statisticsDict:
342 for ampName, stats in statisticsDict["ISR"]["CALIBDIST"].items():
343 if ampName == "detector":
344 nBadColumns = stats[ampName].get("LSST CALIB DEFECTS N_BAD_COLUMNS", np.nan)
345 elif ampName in stats.keys():
346 key = f"LSST CALIB DEFECTS {ampName} N_HOT"
347 rows[ampName][f"{self.config.stageName}_N_HOT"] = stats[ampName].get(key, np.nan)
348 key = f"LSST CALIB DEFECTS {ampName} N_COLD"
349 rows[ampName][f"{self.config.stageName}_N_COLD"] = stats[ampName].get(key, np.nan)
351 # DET results
352 rows["detector"] = rowBase
353 rows["detector"][f"{self.config.stageName}_N_BAD_COLUMNS"] = nBadColumns
355 for ampName, stats in rows.items():
356 rowList.append(stats)
358 return rowList, matrixRowList