Coverage for python/lsst/cp/verify/verifyStats.py : 32%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of cp_verify.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
21import math
23import lsst.afw.geom as afwGeom
24import lsst.afw.math as afwMath
25import lsst.pex.config as pexConfig
26import lsst.pipe.base as pipeBase
27import lsst.pipe.base.connectionTypes as cT
28import lsst.meas.algorithms as measAlg
30from lsst.cp.pipe.cpCombine import vignetteExposure
31from lsst.pipe.tasks.repair import RepairTask
32from .utils import mergeStatDict
34__all__ = ['CpVerifyStatsConfig', 'CpVerifyStatsTask']
37class CpVerifyStatsConnections(pipeBase.PipelineTaskConnections,
38 dimensions={"instrument", "exposure", "detector"},
39 defaultTemplates={}):
40 inputExp = cT.Input(
41 name="postISRCCD",
42 doc="Input exposure to calculate statistics for.",
43 storageClass="Exposure",
44 dimensions=["instrument", "exposure", "detector"],
45 )
46 camera = cT.PrerequisiteInput(
47 name="camera",
48 storageClass="Camera",
49 doc="Input camera.",
50 dimensions=["instrument", ],
51 isCalibration=True,
52 )
54 outputStats = cT.Output(
55 name="detectorStats",
56 doc="Output statistics from cp_verify.",
57 storageClass="StructuredDataDict",
58 dimensions=["instrument", "exposure", "detector"],
59 )
62class CpVerifyStatsConfig(pipeBase.PipelineTaskConfig,
63 pipelineConnections=CpVerifyStatsConnections):
64 """Configuration parameters for CpVerifyStatsTask.
65 """
66 maskNameList = pexConfig.ListField(
67 dtype=str,
68 doc="Mask list to exclude from statistics calculations.",
69 default=['DETECTED', 'BAD', 'NO_DATA'],
70 )
71 doVignette = pexConfig.Field(
72 dtype=bool,
73 doc="Mask vignetted regions?",
74 default=False,
75 )
76 doNormalize = pexConfig.Field(
77 dtype=bool,
78 doc="Normalize by exposure time?",
79 default=False,
80 )
82 # Cosmic ray handling options.
83 doCR = pexConfig.Field(
84 dtype=bool,
85 doc="Run CR rejection?",
86 default=False,
87 )
88 repair = pexConfig.ConfigurableField(
89 target=RepairTask,
90 doc="Repair task to use.",
91 )
92 psfFwhm = pexConfig.Field(
93 dtype=float,
94 default=3.0,
95 doc="Repair PSF FWHM (pixels).",
96 )
97 psfSize = pexConfig.Field(
98 dtype=int,
99 default=21,
100 doc="Repair PSF bounding-box size (pixels).",
101 )
102 crGrow = pexConfig.Field(
103 dtype=int,
104 default=2,
105 doc="Grow radius for CR (pixels).",
106 )
108 # Statistics options.
109 useReadNoise = pexConfig.Field(
110 dtype=bool,
111 doc="Compare sigma against read noise?",
112 default=True,
113 )
114 numSigmaClip = pexConfig.Field(
115 dtype=float,
116 doc="Rejection threshold (sigma) for statistics clipping.",
117 default=5.0,
118 )
119 clipMaxIter = pexConfig.Field(
120 dtype=int,
121 doc="Max number of clipping iterations to apply.",
122 default=3,
123 )
125 # Keywords and statistics to measure from different sources.
126 imageStatKeywords = pexConfig.DictField(
127 keytype=str,
128 itemtype=str,
129 doc="Image statistics to run on amplifier segments.",
130 default={},
131 )
132 unmaskedImageStatKeywords = pexConfig.DictField(
133 keytype=str,
134 itemtype=str,
135 doc="Image statistics to run on amplifier segments, ignoring masks.",
136 default={},
137 )
138 crImageStatKeywords = pexConfig.DictField(
139 keytype=str,
140 itemtype=str,
141 doc="Image statistics to run on CR cleaned amplifier segments.",
142 default={},
143 )
144 normImageStatKeywords = pexConfig.DictField(
145 keytype=str,
146 itemtype=str,
147 doc="Image statistics to run on expTime normalized amplifier segments.",
148 default={},
149 )
150 catalogStatKeywords = pexConfig.DictField(
151 keytype=str,
152 itemtype=str,
153 doc="Statistics to measure from source catalogs of objects in the exposure.",
154 default={},
155 )
156 detectorStatKeywords = pexConfig.DictField(
157 keytype=str,
158 itemtype=str,
159 doc="Statistics to create for the full detector from the per-amplifier measurements.",
160 default={},
161 )
164class CpVerifyStatsTask(pipeBase.PipelineTask, pipeBase.CmdLineTask):
165 """Main statistic measurement and validation class.
167 This operates on a single (exposure, detector) pair, and is
168 designed to be subclassed so specific calibrations can apply their
169 own validation methods.
170 """
171 ConfigClass = CpVerifyStatsConfig
172 _DefaultName = 'cpVerifyStats'
174 def __init__(self, **kwargs):
175 super().__init__(**kwargs)
176 self.makeSubtask("repair")
178 def run(self, inputExp, camera):
179 """Calculate quality statistics and verify they meet the requirements
180 for a calibration.
182 Parameters
183 ----------
184 inputExp : `lsst.afw.image.Exposure`
185 The ISR processed exposure to be measured.
186 camera : `lsst.afw.cameraGeom.Camera`
187 The camera geometry for ``inputExp``.
189 Returns
190 -------
191 result : `lsst.pipe.base.Struct`
192 Result struct with components:
193 - ``outputStats`` : `dict`
194 The output measured statistics.
196 Notes
197 -----
198 The outputStats should have a yaml representation of the form
200 AMP:
201 Amp1:
202 STAT: value
203 STAT2: value2
204 Amp2:
205 Amp3:
206 DET:
207 STAT: value
208 STAT2: value
209 CATALOG:
210 STAT: value
211 STAT2: value
212 VERIFY:
213 DET:
214 TEST: boolean
215 CATALOG:
216 TEST: boolean
217 AMP:
218 Amp1:
219 TEST: boolean
220 TEST2: boolean
221 Amp2:
222 Amp3:
223 SUCCESS: boolean
225 """
226 outputStats = {}
228 if self.config.doVignette:
229 vignetteExposure(inputExp, doUpdateMask=True, maskPlane='NO_DATA',
230 doSetValue=False, log=self.log)
232 mask = inputExp.getMask()
233 maskVal = mask.getPlaneBitMask(self.config.maskNameList)
234 statControl = afwMath.StatisticsControl(self.config.numSigmaClip,
235 self.config.clipMaxIter,
236 maskVal)
238 # This is wrapped below to check for config lengths, as we can
239 # make a number of different image stats.
240 outputStats['AMP'] = self.imageStatistics(inputExp, statControl)
241 if len(self.config.catalogStatKeywords):
242 outputStats['CATALOG'] = self.catalogStatistics(inputExp, statControl)
243 else:
244 outputStats['CATALOG'] = {}
245 if len(self.config.detectorStatKeywords):
246 outputStats['DET'] = self.detectorStatistics(outputStats, statControl)
247 else:
248 outputStats['DET'] = {}
250 outputStats['VERIFY'], outputStats['SUCCESS'] = self.verify(inputExp, outputStats)
252 return pipeBase.Struct(
253 outputStats=outputStats,
254 )
256 @staticmethod
257 def _emptyAmpDict(exposure):
258 """Construct empty dictionary indexed by amplifier names.
260 Parameters
261 ----------
262 exposure : `lsst.afw.image.Exposure`
263 Exposure to extract detector from.
265 Returns
266 -------
267 outputStatistics : `dict` [`str`, `dict`]
268 A skeleton statistics dictionary.
270 Raises
271 ------
272 RuntimeError :
273 Raised if no detector can be found.
274 """
275 outputStatistics = {}
276 detector = exposure.getDetector()
277 if detector is None:
278 raise RuntimeError("No detector found in exposure!")
280 for amp in detector.getAmplifiers():
281 outputStatistics[amp.getName()] = {}
283 return outputStatistics
285 # Image measurement methods.
286 def imageStatistics(self, exposure, statControl):
287 """Measure image statistics for a number of simple image
288 modifications.
290 Parameters
291 ----------
292 exposure : `lsst.afw.image.Exposure`
293 Exposure containing the ISR processed data to measure.
294 statControl : `lsst.afw.math.StatisticsControl`
295 Statistics control object with parameters defined by
296 the config.
298 Returns
299 -------
300 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]]
301 A dictionary indexed by the amplifier name, containing
302 dictionaries of the statistics measured and their values.
304 """
305 outputStatistics = self._emptyAmpDict(exposure)
307 if len(self.config.imageStatKeywords):
308 outputStatistics = mergeStatDict(outputStatistics,
309 self.amplifierStats(exposure,
310 self.config.imageStatKeywords,
311 statControl))
313 if len(self.config.unmaskedImageStatKeywords):
314 outputStatistics = mergeStatDict(outputStatistics, self.unmaskedImageStats(exposure))
316 if len(self.config.normImageStatKeywords):
317 outputStatistics = mergeStatDict(outputStatistics,
318 self.normalizedImageStats(exposure, statControl))
320 if len(self.config.crImageStatKeywords):
321 outputStatistics = mergeStatDict(outputStatistics,
322 self.crImageStats(exposure, statControl))
324 return outputStatistics
326 def amplifierStats(self, exposure, keywordDict, statControl):
327 """Measure amplifier level statistics from the exposure.
329 Parameters
330 ----------
331 exposure : `lsst.afw.image.Exposure`
332 The exposure to measure.
333 keywordDict : `dict` [`str`, `str`]
334 A dictionary of keys to use in the output results, with
335 values the string name associated with the
336 `lsst.afw.math.statistics.Property` to measure.
337 statControl : `lsst.afw.math.StatisticsControl`
338 Statistics control object with parameters defined by
339 the config.
341 Returns
342 -------
343 ampStats : `dict` [`str`, `dict` [`str`, scalar]]
344 A dictionary indexed by the amplifier name, containing
345 dictionaries of the statistics measured and their values.
346 """
347 ampStats = {}
349 # Convert the string names in the keywordDict to the afwMath values.
350 # The statisticToRun is then the bitwise-or of that set.
351 statisticToRun = 0
352 statAccessor = {}
353 for k, v in keywordDict.items():
354 statValue = afwMath.stringToStatisticsProperty(v)
355 statisticToRun |= statValue
356 statAccessor[k] = statValue
358 # Measure stats on all amplifiers.
359 for ampIdx, amp in enumerate(exposure.getDetector()):
360 ampName = amp.getName()
361 theseStats = {}
362 ampExp = exposure.Factory(exposure, amp.getBBox())
363 stats = afwMath.makeStatistics(ampExp.getMaskedImage(), statisticToRun, statControl)
365 for k, v in statAccessor.items():
366 theseStats[k] = stats.getValue(v)
367 ampStats[ampName] = theseStats
369 return ampStats
371 def unmaskedImageStats(self, exposure):
372 """Measure amplifier level statistics on the exposure, including all
373 pixels in the exposure, regardless of any mask planes set.
375 Parameters
376 ----------
377 exposure : `lsst.afw.image.Exposure`
378 The exposure to measure.
380 Returns
381 -------
382 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]]
383 A dictionary indexed by the amplifier name, containing
384 dictionaries of the statistics measured and their values.
385 """
386 noMaskStatsControl = afwMath.StatisticsControl(self.config.numSigmaClip,
387 self.config.clipMaxIter,
388 0x0)
389 return self.amplifierStats(exposure, self.config.unmaskedImageStatKeywords, noMaskStatsControl)
391 def normalizedImageStats(self, exposure, statControl):
392 """Measure amplifier level statistics on the exposure after dividing
393 by the exposure time.
395 Parameters
396 ----------
397 exposure : `lsst.afw.image.Exposure`
398 The exposure to measure.
399 statControl : `lsst.afw.math.StatisticsControl`
400 Statistics control object with parameters defined by
401 the config.
403 Returns
404 -------
405 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]]
406 A dictionary indexed by the amplifier name, containing
407 dictionaries of the statistics measured and their values.
409 Raises
410 ------
411 RuntimeError :
412 Raised if the exposure time cannot be used for normalization.
413 """
414 scaledExposure = exposure.clone()
415 exposureTime = scaledExposure.getInfo().getVisitInfo().getExposureTime()
416 if exposureTime <= 0:
417 raise RuntimeError(f"Invalid exposureTime {exposureTime}.")
418 mi = scaledExposure.getMaskedImage()
419 mi /= exposureTime
421 return self.amplifierStats(scaledExposure, self.config.normImageStatKeywords, statControl)
423 def crImageStats(self, exposure, statControl):
424 """Measure amplifier level statistics on the exposure,
425 after running cosmic ray rejection.
427 Parameters
428 ----------
429 exposure : `lsst.afw.image.Exposure`
430 The exposure to measure.
431 statControl : `lsst.afw.math.StatisticsControl`
432 Statistics control object with parameters defined by
433 the config.
435 Returns
436 -------
437 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]]
438 A dictionary indexed by the amplifier name, containing
439 dictionaries of the statistics measured and their values.
441 """
442 crRejectedExp = exposure.clone()
443 psf = measAlg.SingleGaussianPsf(self.config.psfSize,
444 self.config.psfSize,
445 self.config.psfFwhm/(2*math.sqrt(2*math.log(2))))
446 crRejectedExp.setPsf(psf)
447 self.repair.run(crRejectedExp, keepCRs=False)
448 if self.config.crGrow > 0:
449 crMask = crRejectedExp.getMaskedImage().getMask().getPlaneBitMask("CR")
450 spans = afwGeom.SpanSet.fromMask(crRejectedExp.mask, crMask)
451 spans = spans.dilated(self.config.crGrow)
452 spans.setMask(crRejectedExp.mask, crMask)
454 return self.amplifierStats(crRejectedExp, self.config.crImageStatKeywords, statControl)
456 # Methods that need to be implemented by the calibration-level subclasses.
457 def catalogStatistics(self, exposure, statControl):
458 """Calculate statistics from a catalog.
460 Parameters
461 ----------
462 exposure : `lsst.afw.image.Exposure`
463 The exposure to measure.
464 statControl : `lsst.afw.math.StatisticsControl`
465 Statistics control object with parameters defined by
466 the config.
468 Returns
469 -------
470 outputStatistics : `dict` [`str`, `dict` [`str`, scalar]]
471 A dictionary indexed by the amplifier name, containing
472 dictionaries of the statistics measured and their values.
473 """
474 raise NotImplementedError("Subclasses must implement catalog statistics method.")
476 def detectorStatistics(self, statisticsDict, statControl):
477 """Calculate detector level statistics based on the existing
478 per-amplifier measurements.
480 Parameters
481 ----------
482 statisticsDictionary : `dict` [`str`, `dict` [`str`, scalar]],
483 Dictionary of measured statistics. The inner dictionary
484 should have keys that are statistic names (`str`) with
485 values that are some sort of scalar (`int` or `float` are
486 the mostly likely types).
488 Returns
489 -------
490 outputStatistics : `dict` [`str`, scalar]
491 A dictionary of the statistics measured and their values.
493 Raises
494 ------
495 NotImplementedError :
496 This method must be implemented by the calibration-type
497 subclass.
498 """
499 raise NotImplementedError("Subclasses must implement detector statistics method.")
501 def verify(self, exposure, statisticsDict):
502 """Verify that the measured statistics meet the verification criteria.
504 Parameters
505 ----------
506 exposure : `lsst.afw.image.Exposure`
507 The exposure the statistics are from.
508 statisticsDictionary : `dict` [`str`, `dict` [`str`, scalar]],
509 Dictionary of measured statistics. The inner dictionary
510 should have keys that are statistic names (`str`) with
511 values that are some sort of scalar (`int` or `float` are
512 the mostly likely types).
514 Returns
515 -------
516 outputStatistics : `dict` [`str`, `dict` [`str`, `bool`]]
517 A dictionary indexed by the amplifier name, containing
518 dictionaries of the verification criteria.
519 success : `bool`
520 A boolean indicating whether all tests have passed.
522 Raises
523 ------
524 NotImplementedError :
525 This method must be implemented by the calibration-type
526 subclass.
527 """
528 raise NotImplementedError("Subclasses must implement verification criteria.")