Coverage for python/lsst/pipe/tasks/computeExposureSummaryStats.py: 16%
198 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-04 11:12 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-11-04 11:12 +0000
1# This file is part of pipe_tasks.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22__all__ = ["ComputeExposureSummaryStatsTask", "ComputeExposureSummaryStatsConfig"]
24import warnings
25import numpy as np
26from scipy.stats import median_abs_deviation as sigmaMad
27import astropy.units as units
28from astropy.time import Time
29from astropy.coordinates import AltAz, SkyCoord, EarthLocation
30from lsst.daf.base import DateTime
32import lsst.pipe.base as pipeBase
33import lsst.pex.config as pexConfig
34import lsst.afw.math as afwMath
35import lsst.afw.image as afwImage
36import lsst.geom as geom
37from lsst.utils.timer import timeMethod
40class ComputeExposureSummaryStatsConfig(pexConfig.Config):
41 """Config for ComputeExposureSummaryTask"""
42 sigmaClip = pexConfig.Field(
43 dtype=float,
44 doc="Sigma for outlier rejection for sky noise.",
45 default=3.0,
46 )
47 clipIter = pexConfig.Field(
48 dtype=int,
49 doc="Number of iterations of outlier rejection for sky noise.",
50 default=2,
51 )
52 badMaskPlanes = pexConfig.ListField(
53 dtype=str,
54 doc="Mask planes that, if set, the associated pixel should not be included sky noise calculation.",
55 default=("NO_DATA", "SUSPECT"),
56 )
57 starSelection = pexConfig.Field(
58 doc="Field to select sources to be used in the PSF statistics computation.",
59 dtype=str,
60 default="calib_psf_used"
61 )
62 starShape = pexConfig.Field(
63 doc="Base name of columns to use for the source shape in the PSF statistics computation.",
64 dtype=str,
65 default="slot_Shape"
66 )
67 psfShape = pexConfig.Field(
68 doc="Base name of columns to use for the PSF shape in the PSF statistics computation.",
69 dtype=str,
70 default="slot_PsfShape"
71 )
72 psfSampling = pexConfig.Field(
73 dtype=int,
74 doc="Sampling rate in pixels in each dimension for the maxDistToNearestPsf metric "
75 "caclulation grid (the tradeoff is between adequate sampling versus speed).",
76 default=8,
77 )
78 psfGridSampling = pexConfig.Field(
79 dtype=int,
80 doc="Sampling rate in pixels in each dimension for PSF model robustness metric "
81 "caclulations grid (the tradeoff is between adequate sampling versus speed).",
82 default=96,
83 )
84 psfBadMaskPlanes = pexConfig.ListField(
85 dtype=str,
86 doc="Mask planes that, if set, the associated pixel should not be included in the PSF model "
87 "robutsness metric calculations (namely, maxDistToNearestPsf and psfTraceRadiusDelta).",
88 default=("BAD", "CR", "EDGE", "INTRP", "NO_DATA", "SAT", "SUSPECT"),
89 )
92class ComputeExposureSummaryStatsTask(pipeBase.Task):
93 """Task to compute exposure summary statistics.
95 This task computes various quantities suitable for DPDD and other
96 downstream processing at the detector centers, including:
97 - psfSigma
98 - psfArea
99 - psfIxx
100 - psfIyy
101 - psfIxy
102 - ra
103 - dec
104 - zenithDistance
105 - zeroPoint
106 - skyBg
107 - skyNoise
108 - meanVar
109 - raCorners
110 - decCorners
111 - astromOffsetMean
112 - astromOffsetStd
114 These additional quantities are computed from the stars in the detector:
115 - psfStarDeltaE1Median
116 - psfStarDeltaE2Median
117 - psfStarDeltaE1Scatter
118 - psfStarDeltaE2Scatter
119 - psfStarDeltaSizeMedian
120 - psfStarDeltaSizeScatter
121 - psfStarScaledDeltaSizeScatter
123 These quantities are computed based on the PSF model and image mask
124 to assess the robustness of the PSF model across a given detector
125 (against, e.g., extrapolation instability):
126 - maxDistToNearestPsf
127 - psfTraceRadiusDelta
128 """
129 ConfigClass = ComputeExposureSummaryStatsConfig
130 _DefaultName = "computeExposureSummaryStats"
132 @timeMethod
133 def run(self, exposure, sources, background):
134 """Measure exposure statistics from the exposure, sources, and
135 background.
137 Parameters
138 ----------
139 exposure : `lsst.afw.image.ExposureF`
140 sources : `lsst.afw.table.SourceCatalog`
141 background : `lsst.afw.math.BackgroundList`
143 Returns
144 -------
145 summary : `lsst.afw.image.ExposureSummary`
146 """
147 self.log.info("Measuring exposure statistics")
149 summary = afwImage.ExposureSummaryStats()
151 bbox = exposure.getBBox()
153 psf = exposure.getPsf()
154 self.update_psf_stats(summary, psf, bbox, sources, image_mask=exposure.mask)
156 wcs = exposure.getWcs()
157 visitInfo = exposure.getInfo().getVisitInfo()
158 self.update_wcs_stats(summary, wcs, bbox, visitInfo)
160 photoCalib = exposure.getPhotoCalib()
161 self.update_photo_calib_stats(summary, photoCalib)
163 self.update_background_stats(summary, background)
165 self.update_masked_image_stats(summary, exposure.getMaskedImage())
167 md = exposure.getMetadata()
168 if 'SFM_ASTROM_OFFSET_MEAN' in md:
169 summary.astromOffsetMean = md['SFM_ASTROM_OFFSET_MEAN']
170 summary.astromOffsetStd = md['SFM_ASTROM_OFFSET_STD']
172 return summary
174 def update_psf_stats(self, summary, psf, bbox, sources=None, image_mask=None, sources_is_astropy=False):
175 """Compute all summary-statistic fields that depend on the PSF model.
177 Parameters
178 ----------
179 summary : `lsst.afw.image.ExposureSummaryStats`
180 Summary object to update in-place.
181 psf : `lsst.afw.detection.Psf` or `None`
182 Point spread function model. If `None`, all fields that depend on
183 the PSF will be reset (generally to NaN).
184 bbox : `lsst.geom.Box2I`
185 Bounding box of the image for which summary stats are being
186 computed.
187 sources : `lsst.afw.table.SourceCatalog` or `astropy.table.Table`
188 Catalog for quantities that are computed from source table columns.
189 If `None`, these quantities will be reset (generally to NaN).
190 The type of this table must correspond to the
191 ``sources_is_astropy`` argument.
192 image_mask : `lsst.afw.image.Mask`, optional
193 Mask image that may be used to compute distance-to-nearest-star
194 metrics.
195 sources_is_astropy : `bool`, optional
196 Whether ``sources`` is an `astropy.table.Table` instance instead
197 of an `lsst.afw.table.Catalog` instance. Default is `False` (the
198 latter).
199 """
200 nan = float("nan")
201 summary.psfSigma = nan
202 summary.psfIxx = nan
203 summary.psfIyy = nan
204 summary.psfIxy = nan
205 summary.psfArea = nan
206 summary.nPsfStar = 0
207 summary.psfStarDeltaE1Median = nan
208 summary.psfStarDeltaE2Median = nan
209 summary.psfStarDeltaE1Scatter = nan
210 summary.psfStarDeltaE2Scatter = nan
211 summary.psfStarDeltaSizeMedian = nan
212 summary.psfStarDeltaSizeScatter = nan
213 summary.psfStarScaledDeltaSizeScatter = nan
214 summary.maxDistToNearestPsf = nan
215 summary.psfTraceRadiusDelta = nan
217 if psf is None:
218 return
219 shape = psf.computeShape(bbox.getCenter())
220 summary.psfSigma = shape.getDeterminantRadius()
221 summary.psfIxx = shape.getIxx()
222 summary.psfIyy = shape.getIyy()
223 summary.psfIxy = shape.getIxy()
224 im = psf.computeKernelImage(bbox.getCenter())
225 # The calculation of effective psf area is taken from
226 # meas_base/src/PsfFlux.cc#L112. See
227 # https://github.com/lsst/meas_base/blob/
228 # 750bffe6620e565bda731add1509507f5c40c8bb/src/PsfFlux.cc#L112
229 summary.psfArea = float(np.sum(im.array)/np.sum(im.array**2.))
231 if image_mask is not None:
232 psfTraceRadiusDelta = psf_trace_radius_delta(
233 image_mask,
234 psf,
235 sampling=self.config.psfGridSampling,
236 bad_mask_bits=self.config.psfBadMaskPlanes
237 )
238 summary.psfTraceRadiusDelta = float(psfTraceRadiusDelta)
240 if sources is None:
241 # No sources are available (as in some tests and rare cases where
242 # the selection criteria in finalizeCharacterization lead to no
243 # good sources).
244 return
246 psf_mask = sources[self.config.starSelection] & (~sources[self.config.starShape + '_flag'])
247 nPsfStar = psf_mask.sum()
249 if nPsfStar == 0:
250 # No stars to measure statistics, so we must return the defaults
251 # of 0 stars and NaN values.
252 return
254 if sources_is_astropy:
255 psf_cat = sources[psf_mask]
256 else:
257 psf_cat = sources[psf_mask].copy(deep=True)
259 starXX = psf_cat[self.config.starShape + '_xx']
260 starYY = psf_cat[self.config.starShape + '_yy']
261 starXY = psf_cat[self.config.starShape + '_xy']
262 psfXX = psf_cat[self.config.psfShape + '_xx']
263 psfYY = psf_cat[self.config.psfShape + '_yy']
264 psfXY = psf_cat[self.config.psfShape + '_xy']
266 starSize = (starXX*starYY - starXY**2.)**0.25
267 starE1 = (starXX - starYY)/(starXX + starYY)
268 starE2 = 2*starXY/(starXX + starYY)
269 starSizeMedian = np.median(starSize)
271 psfSize = (psfXX*psfYY - psfXY**2)**0.25
272 psfE1 = (psfXX - psfYY)/(psfXX + psfYY)
273 psfE2 = 2*psfXY/(psfXX + psfYY)
275 psfStarDeltaE1Median = np.median(starE1 - psfE1)
276 psfStarDeltaE1Scatter = sigmaMad(starE1 - psfE1, scale='normal')
277 psfStarDeltaE2Median = np.median(starE2 - psfE2)
278 psfStarDeltaE2Scatter = sigmaMad(starE2 - psfE2, scale='normal')
280 psfStarDeltaSizeMedian = np.median(starSize - psfSize)
281 psfStarDeltaSizeScatter = sigmaMad(starSize - psfSize, scale='normal')
282 psfStarScaledDeltaSizeScatter = psfStarDeltaSizeScatter/starSizeMedian**2.
284 summary.nPsfStar = int(nPsfStar)
285 summary.psfStarDeltaE1Median = float(psfStarDeltaE1Median)
286 summary.psfStarDeltaE2Median = float(psfStarDeltaE2Median)
287 summary.psfStarDeltaE1Scatter = float(psfStarDeltaE1Scatter)
288 summary.psfStarDeltaE2Scatter = float(psfStarDeltaE2Scatter)
289 summary.psfStarDeltaSizeMedian = float(psfStarDeltaSizeMedian)
290 summary.psfStarDeltaSizeScatter = float(psfStarDeltaSizeScatter)
291 summary.psfStarScaledDeltaSizeScatter = float(psfStarScaledDeltaSizeScatter)
293 if image_mask is not None:
294 maxDistToNearestPsf = maximum_nearest_psf_distance(
295 image_mask,
296 psf_cat,
297 sampling=self.config.psfSampling,
298 bad_mask_bits=self.config.psfBadMaskPlanes
299 )
300 summary.maxDistToNearestPsf = float(maxDistToNearestPsf)
302 def update_wcs_stats(self, summary, wcs, bbox, visitInfo):
303 """Compute all summary-statistic fields that depend on the WCS model.
305 Parameters
306 ----------
307 summary : `lsst.afw.image.ExposureSummaryStats`
308 Summary object to update in-place.
309 wcs : `lsst.afw.geom.SkyWcs` or `None`
310 Astrometric calibration model. If `None`, all fields that depend
311 on the WCS will be reset (generally to NaN).
312 bbox : `lsst.geom.Box2I`
313 Bounding box of the image for which summary stats are being
314 computed.
315 visitInfo : `lsst.afw.image.VisitInfo`
316 Observation information used in together with ``wcs`` to compute
317 the zenith distance.
318 """
319 nan = float("nan")
320 summary.raCorners = [nan]*4
321 summary.decCorners = [nan]*4
322 summary.ra = nan
323 summary.dec = nan
324 summary.zenithDistance = nan
326 if wcs is None:
327 return
329 sph_pts = wcs.pixelToSky(geom.Box2D(bbox).getCorners())
330 summary.raCorners = [float(sph.getRa().asDegrees()) for sph in sph_pts]
331 summary.decCorners = [float(sph.getDec().asDegrees()) for sph in sph_pts]
333 sph_pt = wcs.pixelToSky(bbox.getCenter())
334 summary.ra = sph_pt.getRa().asDegrees()
335 summary.dec = sph_pt.getDec().asDegrees()
337 date = visitInfo.getDate()
339 if date.isValid():
340 # We compute the zenithDistance at the center of the detector
341 # rather than use the boresight value available via the visitInfo,
342 # because the zenithDistance may vary significantly over a large
343 # field of view.
344 observatory = visitInfo.getObservatory()
345 loc = EarthLocation(lat=observatory.getLatitude().asDegrees()*units.deg,
346 lon=observatory.getLongitude().asDegrees()*units.deg,
347 height=observatory.getElevation()*units.m)
348 obstime = Time(visitInfo.getDate().get(system=DateTime.MJD),
349 location=loc, format='mjd')
350 coord = SkyCoord(
351 summary.ra*units.degree,
352 summary.dec*units.degree,
353 obstime=obstime,
354 location=loc,
355 )
356 with warnings.catch_warnings():
357 warnings.simplefilter('ignore')
358 altaz = coord.transform_to(AltAz)
360 summary.zenithDistance = float(90.0 - altaz.alt.degree)
362 def update_photo_calib_stats(self, summary, photo_calib):
363 """Compute all summary-statistic fields that depend on the photometric
364 calibration model.
366 Parameters
367 ----------
368 summary : `lsst.afw.image.ExposureSummaryStats`
369 Summary object to update in-place.
370 photo_calib : `lsst.afw.image.PhotoCalib` or `None`
371 Photometric calibration model. If `None`, all fields that depend
372 on the photometric calibration will be reset (generally to NaN).
373 """
374 if photo_calib is not None:
375 summary.zeroPoint = float(2.5*np.log10(photo_calib.getInstFluxAtZeroMagnitude()))
376 else:
377 summary.zeroPoint = float("nan")
379 def update_background_stats(self, summary, background):
380 """Compute summary-statistic fields that depend only on the
381 background model.
383 Parameters
384 ----------
385 summary : `lsst.afw.image.ExposureSummaryStats`
386 Summary object to update in-place.
387 background : `lsst.afw.math.BackgroundList` or `None`
388 Background model. If `None`, all fields that depend on the
389 background will be reset (generally to NaN).
391 Notes
392 -----
393 This does not include fields that depend on the background-subtracted
394 masked image; when the background changes, it should generally be
395 applied to the image and `update_masked_image_stats` should be called
396 as well.
397 """
398 if background is not None:
399 bgStats = (bg[0].getStatsImage().getImage().array
400 for bg in background)
401 summary.skyBg = float(sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats))
402 else:
403 summary.skyBg = float("nan")
405 def update_masked_image_stats(self, summary, masked_image):
406 """Compute summary-statistic fields that depend on the masked image
407 itself.
409 Parameters
410 ----------
411 summary : `lsst.afw.image.ExposureSummaryStats`
412 Summary object to update in-place.
413 masked_image : `lsst.afw.image.MaskedImage` or `None`
414 Masked image. If `None`, all fields that depend
415 on the masked image will be reset (generally to NaN).
416 """
417 nan = float("nan")
418 if masked_image is None:
419 summary.skyNoise = nan
420 summary.meanVar = nan
421 return
422 statsCtrl = afwMath.StatisticsControl()
423 statsCtrl.setNumSigmaClip(self.config.sigmaClip)
424 statsCtrl.setNumIter(self.config.clipIter)
425 statsCtrl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.badMaskPlanes))
426 statsCtrl.setNanSafe(True)
428 statObj = afwMath.makeStatistics(masked_image, afwMath.STDEVCLIP, statsCtrl)
429 skyNoise, _ = statObj.getResult(afwMath.STDEVCLIP)
430 summary.skyNoise = skyNoise
432 statObj = afwMath.makeStatistics(masked_image.variance, masked_image.mask, afwMath.MEANCLIP,
433 statsCtrl)
434 meanVar, _ = statObj.getResult(afwMath.MEANCLIP)
435 summary.meanVar = meanVar
438def maximum_nearest_psf_distance(
439 image_mask,
440 psf_cat,
441 sampling=8,
442 bad_mask_bits=["BAD", "CR", "INTRP", "SAT", "SUSPECT", "NO_DATA", "EDGE"],
443):
444 """Compute the maximum distance of an unmasked pixel to its nearest PSF.
446 Parameters
447 ----------
448 image_mask : `lsst.afw.image.Mask`
449 The mask plane associated with the exposure.
450 psf_cat : `lsst.afw.table.SourceCatalog` or `astropy.table.Table`
451 Catalog containing only the stars used in the PSF modeling.
452 sampling : `int`
453 Sampling rate in each dimension to create the grid of points on which
454 to evaluate the distance to the nearest PSF star. The tradeoff is
455 between adequate sampling versus speed.
456 bad_mask_bits : `list` [`str`]
457 Mask bits required to be absent for a pixel to be considered
458 "unmasked".
460 Returns
461 -------
462 max_dist_to_nearest_psf : `float`
463 The maximum distance (in pixels) of an unmasked pixel to its nearest
464 PSF model star.
465 """
466 mask_arr = image_mask.array[::sampling, ::sampling]
467 bitmask = image_mask.getPlaneBitMask(bad_mask_bits)
468 good = ((mask_arr & bitmask) == 0)
470 x = np.arange(good.shape[1]) * sampling
471 y = np.arange(good.shape[0]) * sampling
472 xx, yy = np.meshgrid(x, y)
474 dist_to_nearest_psf = np.full(good.shape, np.inf)
475 for psf in psf_cat:
476 x_psf = psf["slot_Centroid_x"]
477 y_psf = psf["slot_Centroid_y"]
478 dist_to_nearest_psf = np.minimum(dist_to_nearest_psf, np.hypot(xx - x_psf, yy - y_psf))
479 unmasked_dists = dist_to_nearest_psf * good
480 max_dist_to_nearest_psf = np.max(unmasked_dists)
482 return max_dist_to_nearest_psf
485def psf_trace_radius_delta(
486 image_mask,
487 image_psf,
488 sampling=96,
489 bad_mask_bits=["BAD", "CR", "INTRP", "SAT", "SUSPECT", "NO_DATA", "EDGE"],
490):
491 """Compute the delta between the maximum and minimum model PSF trace radius
492 values evaluated on a grid of points lying in the unmasked region of the
493 image.
495 Parameters
496 ----------
497 image_mask : `lsst.afw.image.Mask`
498 The mask plane associated with the exposure.
499 image_psf : `lsst.afw.detection.Psf`
500 The PSF model associated with the exposure.
501 sampling : `int`
502 Sampling rate in each dimension to create the grid of points at which
503 to evaluate ``image_psf``s trace radius value. The tradeoff is between
504 adequate sampling versus speed.
505 bad_mask_bits : `list` [`str`]
506 Mask bits required to be absent for a pixel to be considered
507 "unmasked".
509 Returns
510 -------
511 psf_trace_radius_delta : `float`
512 The delta (in pixels) between the maximum and minimum model PSF trace
513 radius values evaluated on the x,y-grid subsampled on the unmasked
514 detector pixels by a factor of ``sampling``. If any model PSF trace
515 radius value on the grid evaluates to NaN, then NaN is returned
516 immediately.
517 """
518 psf_trace_radius_list = []
519 mask_arr = image_mask.array[::sampling, ::sampling]
520 bitmask = image_mask.getPlaneBitMask(bad_mask_bits)
521 good = ((mask_arr & bitmask) == 0)
523 x = np.arange(good.shape[1]) * sampling
524 y = np.arange(good.shape[0]) * sampling
525 xx, yy = np.meshgrid(x, y)
527 for x_mesh, y_mesh, good_mesh in zip(xx, yy, good):
528 for x_point, y_point, is_good in zip(x_mesh, y_mesh, good_mesh):
529 if is_good:
530 psf_trace_radius = image_psf.computeShape(geom.Point2D(x_point, y_point)).getTraceRadius()
531 if ~np.isfinite(psf_trace_radius):
532 return float("nan")
533 psf_trace_radius_list.append(psf_trace_radius)
535 psf_trace_radius_delta = np.max(psf_trace_radius_list) - np.min(psf_trace_radius_list)
537 return psf_trace_radius_delta