Coverage for python/lsst/cp/verify/verifyDefects.py: 18%
113 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-10 11:36 +0000
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-10 11:36 +0000
1# This file is part of cp_verify.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21import numpy as np
22import scipy.stats
23import lsst.pipe.base.connectionTypes as cT
24from lsst.ip.isr.isrFunctions import countMaskedPixels
26from .verifyStats import (
27 CpVerifyStatsConfig,
28 CpVerifyStatsTask,
29 CpVerifyStatsConnections,
30)
33__all__ = ["CpVerifyDefectsConfig", "CpVerifyDefectsTask"]
36class CpVerifyDefectsConnections(
37 CpVerifyStatsConnections, dimensions={"instrument", "visit", "detector"}
38):
39 inputExp = cT.Input(
40 name="icExp",
41 doc="Input exposure to calculate statistics for.",
42 storageClass="ExposureF",
43 dimensions=["instrument", "visit", "detector"],
44 )
45 uncorrectedExp = cT.Input(
46 name="uncorrectedExp",
47 doc="Uncorrected input exposure to calculate statistics for.",
48 storageClass="ExposureF",
49 dimensions=["instrument", "visit", "detector"],
50 )
51 inputCatalog = cT.Input(
52 name="icSrc",
53 doc="Input catalog to calculate statistics from.",
54 storageClass="SourceCatalog",
55 dimensions=["instrument", "visit", "detector"],
56 )
57 uncorrectedCatalog = cT.Input(
58 name="uncorrectedSrc",
59 doc="Input catalog without correction applied.",
60 storageClass="SourceCatalog",
61 dimensions=["instrument", "visit", "detector"],
62 )
63 camera = cT.PrerequisiteInput(
64 name="camera",
65 storageClass="Camera",
66 doc="Input camera.",
67 dimensions=["instrument", ],
68 isCalibration=True,
69 )
70 outputStats = cT.Output(
71 name="detectorStats",
72 doc="Output statistics from cp_verify.",
73 storageClass="StructuredDataDict",
74 dimensions=["instrument", "visit", "detector"],
75 )
76 outputResults = cT.Output(
77 name="detectorResults",
78 doc="Output results from cp_verify.",
79 storageClass="ArrowAstropy",
80 dimensions=["instrument", "visit", "detector"],
81 )
82 outputMatrix = cT.Output(
83 name="detectorMatrix",
84 doc="Output matrix results from cp_verify.",
85 storageClass="ArrowAstropy",
86 dimensions=["instrument", "visit", "detector"],
87 )
90class CpVerifyDefectsConfig(
91 CpVerifyStatsConfig, pipelineConnections=CpVerifyDefectsConnections
92):
93 """Inherits from base CpVerifyStatsConfig."""
95 def setDefaults(self):
96 super().setDefaults()
97 self.stageName = 'DEFECTS'
99 self.maskNameList = ["BAD"] # noqa F821
101 self.imageStatKeywords = {
102 "DEFECT_PIXELS": "NMASKED", # noqa F821
103 "OUTLIERS": "NCLIPPED",
104 "MEDIAN": "MEDIAN",
105 "STDEV": "STDEVCLIP",
106 "MIN": "MIN",
107 "MAX": "MAX",
108 }
109 self.unmaskedImageStatKeywords = {
110 "UNMASKED_MIN": "MIN", # noqa F821
111 "UNMASKED_MAX": "MAX",
112 "UNMASKED_STDEV": "STDEVCLIP",
113 "UNMASKED_OUTLIERS": "NCLIPPED",
114 }
115 self.uncorrectedImageStatKeywords = {
116 "UNC_DEFECT_PIXELS": "NMASKED", # noqa F821
117 "UNC_OUTLIERS": "NCLIPPED",
118 "UNC_MEDIAN": "MEDIAN",
119 "UNC_STDEV": "STDEVCLIP",
120 "UNC_MIN": "MIN",
121 "UNC_MAX": "MAX",
122 }
123 # These config options need to have a key/value pair
124 # to run verification analysis, but the contents of
125 # that pair are not used.
126 self.catalogStatKeywords = {"empty": "dictionary"}
127 self.detectorStatKeywords = {"empty": "dictionary"}
130class CpVerifyDefectsTask(CpVerifyStatsTask):
131 """Defects verification sub-class, implementing the verify method.
133 This also applies additional image processing statistics.
134 """
136 ConfigClass = CpVerifyDefectsConfig
137 _DefaultName = "cpVerifyDefects"
139 def catalogStatistics(self, exposure, catalog, uncorrectedCatalog, statControl):
140 """Measure the catalog statistics.
142 Parameters
143 ----------
144 exposure : `lsst.afw.image.Exposure`
145 The exposure to measure.
146 catalog : `lsst.afw.table.Table`
147 The catalog to measure.
148 uncorrectedCatalog : `lsst.afw.table.Table`
149 The uncorrected catalog to measure.
150 statControl : `lsst.afw.math.StatisticsControl`
151 Statistics control object with parameters defined by
152 the config.
154 Returns
155 -------
156 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]]
157 A dictionary indexed by the amplifier name, containing
158 dictionaries of the statistics measured and their values.
160 Notes
161 -----
162 Number of detections test: running with defects would have fewer
163 detections
164 """
165 outputStatistics = {}
167 # Number of detections test
168 outputStatistics["NUM_OBJECTS_BEFORE"] = len(uncorrectedCatalog)
169 outputStatistics["NUM_OBJECTS_AFTER"] = len(catalog)
171 return outputStatistics
173 def detectorStatistics(self, statisticsDict, statControl, exposure=None, uncorrectedExposure=None):
174 """Measure the detector statistics.
176 Parameters
177 ----------
178 statisticsDict : `dict` [`str`, scalar]
179 Dictionary with detector tests.
180 statControl : `lsst.afw.math.StatControl`
181 Statistics control object with parameters defined by
182 the config.
183 exposure : `lsst.afw.image.Exposure`, optional
184 Exposure containing the ISR-processed data to measure.
185 uncorrectedExposure : `lsst.afw.image.Exposure`, optional
186 uncorrected esposure (no defects) containing the
187 ISR-processed data to measure.
189 Returns
190 -------
191 outputStatistics : `dict` [`str`, scalar]
192 A dictionary containing statistics measured and their values.
194 Notes
195 -----
196 Number of cosmic rays test: If there are defects in our data that
197 we didn't properly identify and cover, they might appear similar
198 to cosmic rays because they have sharp edges compared to the point
199 spread function (PSF). When we process the data, if these defects
200 aren't marked in our defect mask, the software might mistakenly think
201 they are cosmic rays and try to remove them. However, if we've already
202 included these defects in the defect mask, the software won't treat
203 them as cosmic rays, so we'll have fewer pixels that are falsely
204 identified and removed as cosmic rays when we compare two sets of
205 data reductions.
206 """
207 outputStatistics = {}
208 # Cosmic Rays test: Count number of cosmic rays before
209 # and after masking with defects
210 nCosmicsBefore = countMaskedPixels(uncorrectedExposure, ["CR"])
211 nCosmicsAfter = countMaskedPixels(exposure, ["CR"])
213 outputStatistics["NUM_COSMICS_BEFORE"] = nCosmicsBefore
214 outputStatistics["NUM_COSMICS_AFTER"] = nCosmicsAfter
216 return outputStatistics
218 def imageStatistics(self, exposure, uncorrectedExposure, statControl):
219 """Measure additional defect statistics.
221 This calls the parent class method first, then adds additional
222 measurements.
224 Parameters
225 ----------
226 exposure : `lsst.afw.image.Exposure`
227 Exposure containing the ISR processed data to measure.
228 statControl : `lsst.afw.math.StatControl`
229 Statistics control object with parameters defined by
230 the config.
232 Returns
233 -------
234 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]]
235 A dictionary indexed by the amplifier name, containing
236 dictionaries of the statistics measured and their values.
238 Notes
239 -----
240 Number of cosmic rays test: If there are defects in our data that
241 we didn't properly identify and cover, they might appear similar
242 to cosmic rays because they have sharp edges compared to the point
243 spread function (PSF). When we process the data, if these defects
244 aren't marked in our defect mask, the software might mistakenly think
245 they are cosmic rays and try to remove them. However, if we've already
246 included these defects in the defect mask, the software won't treat
247 them as cosmic rays, so we'll have fewer pixels that are falsely
248 identified and removed as cosmic rays when we compare two sets of
249 data reductions.
250 """
251 outputStatistics = super().imageStatistics(
252 exposure, uncorrectedExposure, statControl
253 )
255 # Is this a useful test? It saves having to do chi^2 fits,
256 # which are going to be biased by the bulk of points.
257 for amp in exposure.getDetector():
258 ampName = amp.getName()
259 ampExp = exposure.Factory(exposure, amp.getBBox())
261 normImage = ampExp.getImage()
262 normArray = normImage.getArray()
264 normArray -= outputStatistics[ampName]["MEDIAN"]
265 normArray /= outputStatistics[ampName]["STDEV"]
267 probability = scipy.stats.norm.pdf(normArray)
268 outliers = np.where(probability < 1.0 / probability.size, 1.0, 0.0)
269 outputStatistics[ampName]["STAT_OUTLIERS"] = int(np.sum(outliers))
271 return outputStatistics
273 def verify(self, exposure, statisticsDict):
274 """Verify that the measured statistics meet the verification criteria.
276 Parameters
277 ----------
278 exposure : `lsst.afw.image.Exposure`
279 The exposure the statistics are from.
280 statisticsDictionary : `dict` [`str`, `dict` [`str`, scalar]],
281 Dictionary of measured statistics. The inner dictionary
282 should have keys that are statistic names (`str`) with
283 values that are some sort of scalar (`int` or `float` are
284 the mostly likely types).
286 Returns
287 -------
288 outputStatistics : `dict` [`str`, `dict` [`str`, `bool`]]
289 A dictionary indexed by the amplifier name, containing
290 dictionaries of the verification criteria.
291 success : `bool`
292 A boolean indicating if all tests have passed.
293 """
294 # Amplifier statistics
295 ampStats = statisticsDict["AMP"]
296 verifyStats = {}
297 successAmp = True
298 for ampName, stats in ampStats.items():
299 verify = {}
301 # These are not defined in DMTN-101 yet.
302 verify["OUTLIERS"] = bool(stats["UNMASKED_OUTLIERS"] >= stats["OUTLIERS"])
303 verify["STDEV"] = bool(stats["UNMASKED_STDEV"] >= stats["STDEV"])
304 verify["MIN"] = bool(stats["UNMASKED_MIN"] <= stats["MIN"])
305 verify["MAX"] = bool(stats["UNMASKED_MAX"] >= stats["MAX"])
307 # This test is bad, and should be made not bad.
308 verify["PROB_TEST"] = bool(stats["STAT_OUTLIERS"] == stats["DEFECT_PIXELS"])
310 verify["SUCCESS"] = bool(np.all(list(verify.values())))
311 if verify["SUCCESS"] is False:
312 successAmp = False
314 verifyStats[ampName] = verify
316 # Detector statistics
317 detStats = statisticsDict["DET"]
318 verifyStatsDet = {}
319 successDet = True
320 # Cosmic rays test from DM-38563, before and after defects.
321 verifyStatsDet["NUMBER_COSMIC_RAYS"] = bool(
322 detStats["NUM_COSMICS_BEFORE"] > detStats["NUM_COSMICS_AFTER"]
323 )
325 verifyStatsDet["SUCCESS"] = bool(np.all(list(verifyStatsDet.values())))
326 if verifyStatsDet["SUCCESS"] is False:
327 successDet = False
329 # Catalog statistics
330 catStats = statisticsDict["CATALOG"]
331 verifyStatsCat = {}
332 successCat = True
333 # Detection tests from DM-38563, before and after defects.
334 verifyStatsCat["NUMBER_DETECTIONS"] = bool(
335 catStats["NUM_OBJECTS_BEFORE"] > catStats["NUM_OBJECTS_AFTER"]
336 )
338 verifyStatsCat["SUCCESS"] = bool(np.all(list(verifyStatsCat.values())))
339 if verifyStatsCat["SUCCESS"] is False:
340 successCat = False
342 success = successDet & successAmp & successCat
343 return {
344 "AMP": verifyStats,
345 "DET": verifyStatsDet,
346 "CATALOG": verifyStatsCat,
347 }, bool(success)
349 def repackStats(self, statisticsDict, dimensions):
350 # docstring inherited
351 rows = {}
352 rowList = []
353 matrixRowList = None
355 if self.config.useIsrStatistics:
356 mjd = statisticsDict["ISR"]["MJD"]
357 else:
358 mjd = np.nan
360 rowBase = {
361 "instrument": dimensions["instrument"],
362 "exposure": dimensions["visit"], # ensure an exposure dimension for downstream.
363 "visit": dimensions["visit"],
364 "detector": dimensions["detector"],
365 "mjd": mjd,
366 }
368 # AMP results
369 for ampName, stats in statisticsDict["AMP"].items():
370 rows[ampName] = {}
371 rows[ampName].update(rowBase)
372 rows[ampName]["amplifier"] = ampName
373 for key, value in stats.items():
374 rows[ampName][f"{self.config.stageName}_{key}"] = value
376 # ISR results
377 nBadColumns = np.nan
378 if self.config.useIsrStatistics and "ISR" in statisticsDict:
379 for ampName, stats in statisticsDict["ISR"]["CALIBDIST"].items():
380 if ampName == "detector":
381 nBadColumns = stats[ampName].get("LSST CALIB DEFECTS N_BAD_COLUMNS", np.nan)
382 elif ampName in stats.keys():
383 key = f"LSST CALIB DEFECTS {ampName} N_HOT"
384 rows[ampName][f"{self.config.stageName}_N_HOT"] = stats[ampName].get(key, np.nan)
385 key = f"LSST CALIB DEFECTS {ampName} N_COLD"
386 rows[ampName][f"{self.config.stageName}_N_COLD"] = stats[ampName].get(key, np.nan)
388 # DET results
389 rows["detector"] = rowBase
390 rows["detector"][f"{self.config.stageName}_N_BAD_COLUMNS"] = nBadColumns
392 for ampName, stats in rows.items():
393 rowList.append(stats)
395 return rowList, matrixRowList