Coverage for python/lsst/cp/verify/verifyCalib.py: 35%
70 statements
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-10 11:36 +0000
« prev ^ index » next coverage.py v7.5.1, created at 2024-05-10 11:36 +0000
1# This file is part of cp_verify.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21import numpy as np
22from astropy.table import Table
24import lsst.pex.config as pexConfig
25import lsst.pipe.base as pipeBase
26import lsst.pipe.base.connectionTypes as cT
28__all__ = ['CpVerifyCalibConfig', 'CpVerifyCalibTask']
31class CpVerifyCalibConnections(pipeBase.PipelineTaskConnections,
32 dimensions={"instrument", "detector"},
33 defaultTemplates={}):
35 exposure = cT.Input(
36 name="raw",
37 doc="Exposure to retreve calibration",
38 storageClass='Exposure',
39 dimensions=("instrument", "detector", "exposure"),
40 multiple=True,
41 deferLoad=True,
42 )
44 inputCalib = cT.PrerequisiteInput(
45 name="calib",
46 doc="Input calib to calculate statistics for.",
47 storageClass="IsrCalib",
48 dimensions=["instrument", "detector"],
49 isCalibration=True
50 )
52 camera = cT.PrerequisiteInput(
53 name="camera",
54 doc="Input camera to use for gain lookup.",
55 storageClass="Camera",
56 dimensions=("instrument",),
57 isCalibration=True,
58 )
60 outputStats = cT.Output(
61 name="calibStats",
62 doc="Output statistics from cp_verify.",
63 storageClass="StructuredDataDict",
64 dimensions=["instrument", "detector"],
65 )
66 outputResults = cT.Output(
67 name="detectorResults",
68 doc="Output results from cp_verify.",
69 storageClass="ArrowAstropy",
70 dimensions=["instrument", "detector"],
71 )
72 outputMatrix = cT.Output(
73 name="detectorMatrix",
74 doc="Output matrix results from cp_verify.",
75 storageClass="ArrowAstropy",
76 dimensions=["instrument", "detector"],
77 )
79 def __init__(self, *, config=None):
80 super().__init__(config=config)
82 if not config.hasMatrixCatalog:
83 self.outputs.discard("outputMatrix")
86class CpVerifyCalibConfig(pipeBase.PipelineTaskConfig,
87 pipelineConnections=CpVerifyCalibConnections):
88 """Configuration parameters for CpVerifyCalibTask.
89 """
90 # Statistics options.
91 useReadNoise = pexConfig.Field(
92 dtype=bool,
93 doc="Compare sigma against read noise?",
94 default=True,
95 )
96 numSigmaClip = pexConfig.Field(
97 dtype=float,
98 doc="Rejection threshold (sigma) for statistics clipping.",
99 default=5.0,
100 )
101 clipMaxIter = pexConfig.Field(
102 dtype=int,
103 doc="Max number of clipping iterations to apply.",
104 default=3,
105 )
107 # Keywords and statistics to measure from different sources.
108 calibStatKeywords = pexConfig.DictField(
109 keytype=str,
110 itemtype=str,
111 doc="Calib statistics to run.",
112 default={},
113 )
115 stageName = pexConfig.Field(
116 dtype=str,
117 doc="Stage name to use in columns.",
118 default="NOCALIB",
119 )
120 useIsrStatistics = pexConfig.Field(
121 dtype=bool,
122 doc="Use statistics calculated by IsrTask?",
123 default=False,
124 )
125 hasMatrixCatalog = pexConfig.Field(
126 dtype=bool,
127 doc="Will a matrix table of results be made?",
128 default=False,
129 )
132class CpVerifyCalibTask(pipeBase.PipelineTask):
133 """Main statistic measurement and validation class.
135 This operates on a generic calibration, and is designed to be
136 subclassed so specific calibrations can apply their own validation
137 methods.
138 """
140 ConfigClass = CpVerifyCalibConfig
141 _DefaultName = 'cpVerifyCalib'
143 def runQuantum(self, butlerQC, inputRefs, outputRefs):
144 inputs = butlerQC.get(inputRefs)
145 inputs["dimensions"] = dict(inputRefs.inputCalib.dataId.required)
147 outputs = self.run(**inputs)
148 butlerQC.put(outputs, outputRefs)
150 def run(self,
151 inputCalib,
152 camera=None,
153 exposure=None,
154 dimensions=None,
155 ):
156 """Calculate quality statistics and verify they meet the requirements
157 for a calibration.
159 Parameters
160 ----------
161 inputCalib : `lsst.ip.isr.IsrCalib`
162 The calibration to be measured.
163 camera : `lsst.afw.cameraGeom.Camera`, optional
164 Input camera.
165 exposure : `lsst.afw.image.Exposure`, optional
166 Dummy exposure to identify a particular calibration
167 dataset.
168 dimensions : `dict`
169 Dictionary of input dictionary.
171 Returns
172 -------
173 result : `lsst.pipe.base.Struct`
174 Result struct with components:
175 - ``outputStats`` : `dict`
176 The output measured statistics.
177 """
178 outputStats = {}
179 outputStats['AMP'] = self.amplifierStatistics(inputCalib, camera=camera)
180 outputStats['DET'] = self.detectorStatistics(inputCalib, camera=camera)
181 outputStats['VERIFY'], outputStats['SUCCESS'] = self.verify(inputCalib, outputStats, camera=camera)
183 outputResults, outputMatrix = self.repackStats(outputStats, dimensions)
184 if outputResults is not None:
185 outputResults = Table(outputResults)
186 if outputMatrix is not None:
187 outputMatrix = Table(outputMatrix)
189 return pipeBase.Struct(
190 outputStats=outputStats,
191 outputResults=outputResults,
192 outputMatrix=outputMatrix,
193 )
195 # Methods that need to be implemented by the calibration-level subclasses.
196 def detectorStatistics(self, inputCalib, camera=None, exposure=None):
197 """Calculate detector level statistics from the calibration.
199 Parameters
200 ----------
201 inputCalib : `lsst.ip.isr.IsrCalib`
202 The calibration to verify.
204 Returns
205 -------
206 outputStatistics : `dict` [`str`, scalar]
207 A dictionary of the statistics measured and their values.
208 camera : `lsst.afw.cameraGeom.Camera`, optional
209 Input camera.
210 exposure : `lsst.afw.image.Exposure`, optional
211 Dummy exposure to identify a particular calibration
212 dataset.
214 Raises
215 ------
216 NotImplementedError :
217 This method must be implemented by the calibration-type
218 subclass.
219 """
220 raise NotImplementedError("Subclasses must implement detector statistics method.")
222 def amplifierStatistics(self, inputCalib, camera=None, exposure=None):
223 """Calculate amplifier level statistics from the calibration.
225 Parameters
226 ----------
227 inputCalib : `lsst.ip.isr.IsrCalib`
228 The calibration to verify.
229 camera : `lsst.afw.cameraGeom.Camera`, optional
230 Input camera.
231 exposure : `lsst.afw.image.Exposure`, optional
232 Dummy exposure to identify a particular calibration
233 dataset.
235 Returns
236 -------
237 outputStatistics : `dict` [`str`, scalar]
238 A dictionary of the statistics measured and their values.
240 Raises
241 ------
242 NotImplementedError :
243 This method must be implemented by the calibration-type
244 subclass.
245 """
246 raise NotImplementedError("Subclasses must implement amplifier statistics method.")
248 def verify(self, inputCalib, statisticsDict, camera=None, exposure=None):
249 """Verify that the measured calibration meet the verification criteria.
251 Parameters
252 ----------
253 inputCalib : `lsst.ip.isr.IsrCalib`
254 The calibration to verify.
255 statisticsDictionary : `dict` [`str`, `dict` [`str`, scalar]],
256 Dictionary of measured statistics. The inner dictionary
257 should have keys that are statistic names (`str`) with
258 values that are some sort of scalar (`int` or `float` are
259 the mostly likely types).
260 camera : `lsst.afw.cameraGeom.Camera`, optional
261 Input camera.
262 exposure : `lsst.afw.image.Exposure`, optional
263 Dummy exposure to identify a particular calibration
264 dataset.
266 Returns
267 -------
268 outputStatistics : `dict` [`str`, `dict` [`str`, `bool`]]
269 A dictionary indexed by the amplifier name, containing
270 dictionaries of the verification criteria.
271 success : `bool`
272 A boolean indicating whether all tests have passed.
274 Raises
275 ------
276 NotImplementedError :
277 This method must be implemented by the calibration-type
278 subclass.
279 """
280 raise NotImplementedError("Subclasses must implement verification criteria.")
282 def repackStats(self, statisticsDict, dimensions):
283 """Repack information into flat tables.
285 This method may be redefined in subclasses. This default
286 version will repack simple amp-level statistics and
287 verification results.
289 Parameters
290 ----------
291 statisticsDictionary : `dict` [`str`, `dict` [`str`, scalar]],
292 Dictionary of measured statistics. The inner dictionary
293 should have keys that are statistic names (`str`) with
294 values that are some sort of scalar (`int` or `float` are
295 the mostly likely types).
296 dimensions : `dict`
297 The dictionary of dimensions values for this data, to be
298 included in the output results.
300 Returns
301 -------
302 outputResults : `list` [`dict`]
303 A list of rows to add to the output table.
304 outputMatrix : `list` [`dict`]
305 A list of rows to add to the output matrix.
306 """
307 rows = {}
308 rowList = []
309 matrixRowList = None
311 if self.config.useIsrStatistics:
312 mjd = statisticsDict["ISR"]["MJD"]
313 else:
314 mjd = np.nan
316 rowBase = {
317 "instrument": dimensions["instrument"],
318 "detector": dimensions["detector"],
319 "mjd": mjd,
320 }
322 # AMP results:
323 for ampName, stats in statisticsDict["AMP"].items():
324 rows[ampName] = {}
325 rows[ampName].update(rowBase)
326 rows[ampName]["amplifier"] = ampName
327 for key, value in stats.items():
328 rows[ampName][f"{self.config.stageName}_{key}"] = value
330 # VERIFY results
331 for ampName, stats in statisticsDict["VERIFY"]["AMP"].items():
332 for key, value in stats.items():
333 rows[ampName][f"{self.config.stageName}_VERIFY_{key}"] = value
335 # pack final list
336 for ampName, stats in rows.items():
337 rowList.append(stats)
339 return rowList, matrixRowList