Coverage for python/lsst/pipe/tasks/computeExposureSummaryStats.py: 16%

198 statements  

« prev     ^ index     » next       coverage.py v7.3.2, created at 2023-11-04 11:12 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22__all__ = ["ComputeExposureSummaryStatsTask", "ComputeExposureSummaryStatsConfig"] 

23 

24import warnings 

25import numpy as np 

26from scipy.stats import median_abs_deviation as sigmaMad 

27import astropy.units as units 

28from astropy.time import Time 

29from astropy.coordinates import AltAz, SkyCoord, EarthLocation 

30from lsst.daf.base import DateTime 

31 

32import lsst.pipe.base as pipeBase 

33import lsst.pex.config as pexConfig 

34import lsst.afw.math as afwMath 

35import lsst.afw.image as afwImage 

36import lsst.geom as geom 

37from lsst.utils.timer import timeMethod 

38 

39 

40class ComputeExposureSummaryStatsConfig(pexConfig.Config): 

41 """Config for ComputeExposureSummaryTask""" 

42 sigmaClip = pexConfig.Field( 

43 dtype=float, 

44 doc="Sigma for outlier rejection for sky noise.", 

45 default=3.0, 

46 ) 

47 clipIter = pexConfig.Field( 

48 dtype=int, 

49 doc="Number of iterations of outlier rejection for sky noise.", 

50 default=2, 

51 ) 

52 badMaskPlanes = pexConfig.ListField( 

53 dtype=str, 

54 doc="Mask planes that, if set, the associated pixel should not be included sky noise calculation.", 

55 default=("NO_DATA", "SUSPECT"), 

56 ) 

57 starSelection = pexConfig.Field( 

58 doc="Field to select sources to be used in the PSF statistics computation.", 

59 dtype=str, 

60 default="calib_psf_used" 

61 ) 

62 starShape = pexConfig.Field( 

63 doc="Base name of columns to use for the source shape in the PSF statistics computation.", 

64 dtype=str, 

65 default="slot_Shape" 

66 ) 

67 psfShape = pexConfig.Field( 

68 doc="Base name of columns to use for the PSF shape in the PSF statistics computation.", 

69 dtype=str, 

70 default="slot_PsfShape" 

71 ) 

72 psfSampling = pexConfig.Field( 

73 dtype=int, 

74 doc="Sampling rate in pixels in each dimension for the maxDistToNearestPsf metric " 

75 "caclulation grid (the tradeoff is between adequate sampling versus speed).", 

76 default=8, 

77 ) 

78 psfGridSampling = pexConfig.Field( 

79 dtype=int, 

80 doc="Sampling rate in pixels in each dimension for PSF model robustness metric " 

81 "caclulations grid (the tradeoff is between adequate sampling versus speed).", 

82 default=96, 

83 ) 

84 psfBadMaskPlanes = pexConfig.ListField( 

85 dtype=str, 

86 doc="Mask planes that, if set, the associated pixel should not be included in the PSF model " 

87 "robutsness metric calculations (namely, maxDistToNearestPsf and psfTraceRadiusDelta).", 

88 default=("BAD", "CR", "EDGE", "INTRP", "NO_DATA", "SAT", "SUSPECT"), 

89 ) 

90 

91 

92class ComputeExposureSummaryStatsTask(pipeBase.Task): 

93 """Task to compute exposure summary statistics. 

94 

95 This task computes various quantities suitable for DPDD and other 

96 downstream processing at the detector centers, including: 

97 - psfSigma 

98 - psfArea 

99 - psfIxx 

100 - psfIyy 

101 - psfIxy 

102 - ra 

103 - dec 

104 - zenithDistance 

105 - zeroPoint 

106 - skyBg 

107 - skyNoise 

108 - meanVar 

109 - raCorners 

110 - decCorners 

111 - astromOffsetMean 

112 - astromOffsetStd 

113 

114 These additional quantities are computed from the stars in the detector: 

115 - psfStarDeltaE1Median 

116 - psfStarDeltaE2Median 

117 - psfStarDeltaE1Scatter 

118 - psfStarDeltaE2Scatter 

119 - psfStarDeltaSizeMedian 

120 - psfStarDeltaSizeScatter 

121 - psfStarScaledDeltaSizeScatter 

122 

123 These quantities are computed based on the PSF model and image mask 

124 to assess the robustness of the PSF model across a given detector 

125 (against, e.g., extrapolation instability): 

126 - maxDistToNearestPsf 

127 - psfTraceRadiusDelta 

128 """ 

129 ConfigClass = ComputeExposureSummaryStatsConfig 

130 _DefaultName = "computeExposureSummaryStats" 

131 

132 @timeMethod 

133 def run(self, exposure, sources, background): 

134 """Measure exposure statistics from the exposure, sources, and 

135 background. 

136 

137 Parameters 

138 ---------- 

139 exposure : `lsst.afw.image.ExposureF` 

140 sources : `lsst.afw.table.SourceCatalog` 

141 background : `lsst.afw.math.BackgroundList` 

142 

143 Returns 

144 ------- 

145 summary : `lsst.afw.image.ExposureSummary` 

146 """ 

147 self.log.info("Measuring exposure statistics") 

148 

149 summary = afwImage.ExposureSummaryStats() 

150 

151 bbox = exposure.getBBox() 

152 

153 psf = exposure.getPsf() 

154 self.update_psf_stats(summary, psf, bbox, sources, image_mask=exposure.mask) 

155 

156 wcs = exposure.getWcs() 

157 visitInfo = exposure.getInfo().getVisitInfo() 

158 self.update_wcs_stats(summary, wcs, bbox, visitInfo) 

159 

160 photoCalib = exposure.getPhotoCalib() 

161 self.update_photo_calib_stats(summary, photoCalib) 

162 

163 self.update_background_stats(summary, background) 

164 

165 self.update_masked_image_stats(summary, exposure.getMaskedImage()) 

166 

167 md = exposure.getMetadata() 

168 if 'SFM_ASTROM_OFFSET_MEAN' in md: 

169 summary.astromOffsetMean = md['SFM_ASTROM_OFFSET_MEAN'] 

170 summary.astromOffsetStd = md['SFM_ASTROM_OFFSET_STD'] 

171 

172 return summary 

173 

174 def update_psf_stats(self, summary, psf, bbox, sources=None, image_mask=None, sources_is_astropy=False): 

175 """Compute all summary-statistic fields that depend on the PSF model. 

176 

177 Parameters 

178 ---------- 

179 summary : `lsst.afw.image.ExposureSummaryStats` 

180 Summary object to update in-place. 

181 psf : `lsst.afw.detection.Psf` or `None` 

182 Point spread function model. If `None`, all fields that depend on 

183 the PSF will be reset (generally to NaN). 

184 bbox : `lsst.geom.Box2I` 

185 Bounding box of the image for which summary stats are being 

186 computed. 

187 sources : `lsst.afw.table.SourceCatalog` or `astropy.table.Table` 

188 Catalog for quantities that are computed from source table columns. 

189 If `None`, these quantities will be reset (generally to NaN). 

190 The type of this table must correspond to the 

191 ``sources_is_astropy`` argument. 

192 image_mask : `lsst.afw.image.Mask`, optional 

193 Mask image that may be used to compute distance-to-nearest-star 

194 metrics. 

195 sources_is_astropy : `bool`, optional 

196 Whether ``sources`` is an `astropy.table.Table` instance instead 

197 of an `lsst.afw.table.Catalog` instance. Default is `False` (the 

198 latter). 

199 """ 

200 nan = float("nan") 

201 summary.psfSigma = nan 

202 summary.psfIxx = nan 

203 summary.psfIyy = nan 

204 summary.psfIxy = nan 

205 summary.psfArea = nan 

206 summary.nPsfStar = 0 

207 summary.psfStarDeltaE1Median = nan 

208 summary.psfStarDeltaE2Median = nan 

209 summary.psfStarDeltaE1Scatter = nan 

210 summary.psfStarDeltaE2Scatter = nan 

211 summary.psfStarDeltaSizeMedian = nan 

212 summary.psfStarDeltaSizeScatter = nan 

213 summary.psfStarScaledDeltaSizeScatter = nan 

214 summary.maxDistToNearestPsf = nan 

215 summary.psfTraceRadiusDelta = nan 

216 

217 if psf is None: 

218 return 

219 shape = psf.computeShape(bbox.getCenter()) 

220 summary.psfSigma = shape.getDeterminantRadius() 

221 summary.psfIxx = shape.getIxx() 

222 summary.psfIyy = shape.getIyy() 

223 summary.psfIxy = shape.getIxy() 

224 im = psf.computeKernelImage(bbox.getCenter()) 

225 # The calculation of effective psf area is taken from 

226 # meas_base/src/PsfFlux.cc#L112. See 

227 # https://github.com/lsst/meas_base/blob/ 

228 # 750bffe6620e565bda731add1509507f5c40c8bb/src/PsfFlux.cc#L112 

229 summary.psfArea = float(np.sum(im.array)/np.sum(im.array**2.)) 

230 

231 if image_mask is not None: 

232 psfTraceRadiusDelta = psf_trace_radius_delta( 

233 image_mask, 

234 psf, 

235 sampling=self.config.psfGridSampling, 

236 bad_mask_bits=self.config.psfBadMaskPlanes 

237 ) 

238 summary.psfTraceRadiusDelta = float(psfTraceRadiusDelta) 

239 

240 if sources is None: 

241 # No sources are available (as in some tests and rare cases where 

242 # the selection criteria in finalizeCharacterization lead to no 

243 # good sources). 

244 return 

245 

246 psf_mask = sources[self.config.starSelection] & (~sources[self.config.starShape + '_flag']) 

247 nPsfStar = psf_mask.sum() 

248 

249 if nPsfStar == 0: 

250 # No stars to measure statistics, so we must return the defaults 

251 # of 0 stars and NaN values. 

252 return 

253 

254 if sources_is_astropy: 

255 psf_cat = sources[psf_mask] 

256 else: 

257 psf_cat = sources[psf_mask].copy(deep=True) 

258 

259 starXX = psf_cat[self.config.starShape + '_xx'] 

260 starYY = psf_cat[self.config.starShape + '_yy'] 

261 starXY = psf_cat[self.config.starShape + '_xy'] 

262 psfXX = psf_cat[self.config.psfShape + '_xx'] 

263 psfYY = psf_cat[self.config.psfShape + '_yy'] 

264 psfXY = psf_cat[self.config.psfShape + '_xy'] 

265 

266 starSize = (starXX*starYY - starXY**2.)**0.25 

267 starE1 = (starXX - starYY)/(starXX + starYY) 

268 starE2 = 2*starXY/(starXX + starYY) 

269 starSizeMedian = np.median(starSize) 

270 

271 psfSize = (psfXX*psfYY - psfXY**2)**0.25 

272 psfE1 = (psfXX - psfYY)/(psfXX + psfYY) 

273 psfE2 = 2*psfXY/(psfXX + psfYY) 

274 

275 psfStarDeltaE1Median = np.median(starE1 - psfE1) 

276 psfStarDeltaE1Scatter = sigmaMad(starE1 - psfE1, scale='normal') 

277 psfStarDeltaE2Median = np.median(starE2 - psfE2) 

278 psfStarDeltaE2Scatter = sigmaMad(starE2 - psfE2, scale='normal') 

279 

280 psfStarDeltaSizeMedian = np.median(starSize - psfSize) 

281 psfStarDeltaSizeScatter = sigmaMad(starSize - psfSize, scale='normal') 

282 psfStarScaledDeltaSizeScatter = psfStarDeltaSizeScatter/starSizeMedian**2. 

283 

284 summary.nPsfStar = int(nPsfStar) 

285 summary.psfStarDeltaE1Median = float(psfStarDeltaE1Median) 

286 summary.psfStarDeltaE2Median = float(psfStarDeltaE2Median) 

287 summary.psfStarDeltaE1Scatter = float(psfStarDeltaE1Scatter) 

288 summary.psfStarDeltaE2Scatter = float(psfStarDeltaE2Scatter) 

289 summary.psfStarDeltaSizeMedian = float(psfStarDeltaSizeMedian) 

290 summary.psfStarDeltaSizeScatter = float(psfStarDeltaSizeScatter) 

291 summary.psfStarScaledDeltaSizeScatter = float(psfStarScaledDeltaSizeScatter) 

292 

293 if image_mask is not None: 

294 maxDistToNearestPsf = maximum_nearest_psf_distance( 

295 image_mask, 

296 psf_cat, 

297 sampling=self.config.psfSampling, 

298 bad_mask_bits=self.config.psfBadMaskPlanes 

299 ) 

300 summary.maxDistToNearestPsf = float(maxDistToNearestPsf) 

301 

302 def update_wcs_stats(self, summary, wcs, bbox, visitInfo): 

303 """Compute all summary-statistic fields that depend on the WCS model. 

304 

305 Parameters 

306 ---------- 

307 summary : `lsst.afw.image.ExposureSummaryStats` 

308 Summary object to update in-place. 

309 wcs : `lsst.afw.geom.SkyWcs` or `None` 

310 Astrometric calibration model. If `None`, all fields that depend 

311 on the WCS will be reset (generally to NaN). 

312 bbox : `lsst.geom.Box2I` 

313 Bounding box of the image for which summary stats are being 

314 computed. 

315 visitInfo : `lsst.afw.image.VisitInfo` 

316 Observation information used in together with ``wcs`` to compute 

317 the zenith distance. 

318 """ 

319 nan = float("nan") 

320 summary.raCorners = [nan]*4 

321 summary.decCorners = [nan]*4 

322 summary.ra = nan 

323 summary.dec = nan 

324 summary.zenithDistance = nan 

325 

326 if wcs is None: 

327 return 

328 

329 sph_pts = wcs.pixelToSky(geom.Box2D(bbox).getCorners()) 

330 summary.raCorners = [float(sph.getRa().asDegrees()) for sph in sph_pts] 

331 summary.decCorners = [float(sph.getDec().asDegrees()) for sph in sph_pts] 

332 

333 sph_pt = wcs.pixelToSky(bbox.getCenter()) 

334 summary.ra = sph_pt.getRa().asDegrees() 

335 summary.dec = sph_pt.getDec().asDegrees() 

336 

337 date = visitInfo.getDate() 

338 

339 if date.isValid(): 

340 # We compute the zenithDistance at the center of the detector 

341 # rather than use the boresight value available via the visitInfo, 

342 # because the zenithDistance may vary significantly over a large 

343 # field of view. 

344 observatory = visitInfo.getObservatory() 

345 loc = EarthLocation(lat=observatory.getLatitude().asDegrees()*units.deg, 

346 lon=observatory.getLongitude().asDegrees()*units.deg, 

347 height=observatory.getElevation()*units.m) 

348 obstime = Time(visitInfo.getDate().get(system=DateTime.MJD), 

349 location=loc, format='mjd') 

350 coord = SkyCoord( 

351 summary.ra*units.degree, 

352 summary.dec*units.degree, 

353 obstime=obstime, 

354 location=loc, 

355 ) 

356 with warnings.catch_warnings(): 

357 warnings.simplefilter('ignore') 

358 altaz = coord.transform_to(AltAz) 

359 

360 summary.zenithDistance = float(90.0 - altaz.alt.degree) 

361 

362 def update_photo_calib_stats(self, summary, photo_calib): 

363 """Compute all summary-statistic fields that depend on the photometric 

364 calibration model. 

365 

366 Parameters 

367 ---------- 

368 summary : `lsst.afw.image.ExposureSummaryStats` 

369 Summary object to update in-place. 

370 photo_calib : `lsst.afw.image.PhotoCalib` or `None` 

371 Photometric calibration model. If `None`, all fields that depend 

372 on the photometric calibration will be reset (generally to NaN). 

373 """ 

374 if photo_calib is not None: 

375 summary.zeroPoint = float(2.5*np.log10(photo_calib.getInstFluxAtZeroMagnitude())) 

376 else: 

377 summary.zeroPoint = float("nan") 

378 

379 def update_background_stats(self, summary, background): 

380 """Compute summary-statistic fields that depend only on the 

381 background model. 

382 

383 Parameters 

384 ---------- 

385 summary : `lsst.afw.image.ExposureSummaryStats` 

386 Summary object to update in-place. 

387 background : `lsst.afw.math.BackgroundList` or `None` 

388 Background model. If `None`, all fields that depend on the 

389 background will be reset (generally to NaN). 

390 

391 Notes 

392 ----- 

393 This does not include fields that depend on the background-subtracted 

394 masked image; when the background changes, it should generally be 

395 applied to the image and `update_masked_image_stats` should be called 

396 as well. 

397 """ 

398 if background is not None: 

399 bgStats = (bg[0].getStatsImage().getImage().array 

400 for bg in background) 

401 summary.skyBg = float(sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats)) 

402 else: 

403 summary.skyBg = float("nan") 

404 

405 def update_masked_image_stats(self, summary, masked_image): 

406 """Compute summary-statistic fields that depend on the masked image 

407 itself. 

408 

409 Parameters 

410 ---------- 

411 summary : `lsst.afw.image.ExposureSummaryStats` 

412 Summary object to update in-place. 

413 masked_image : `lsst.afw.image.MaskedImage` or `None` 

414 Masked image. If `None`, all fields that depend 

415 on the masked image will be reset (generally to NaN). 

416 """ 

417 nan = float("nan") 

418 if masked_image is None: 

419 summary.skyNoise = nan 

420 summary.meanVar = nan 

421 return 

422 statsCtrl = afwMath.StatisticsControl() 

423 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

424 statsCtrl.setNumIter(self.config.clipIter) 

425 statsCtrl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.badMaskPlanes)) 

426 statsCtrl.setNanSafe(True) 

427 

428 statObj = afwMath.makeStatistics(masked_image, afwMath.STDEVCLIP, statsCtrl) 

429 skyNoise, _ = statObj.getResult(afwMath.STDEVCLIP) 

430 summary.skyNoise = skyNoise 

431 

432 statObj = afwMath.makeStatistics(masked_image.variance, masked_image.mask, afwMath.MEANCLIP, 

433 statsCtrl) 

434 meanVar, _ = statObj.getResult(afwMath.MEANCLIP) 

435 summary.meanVar = meanVar 

436 

437 

438def maximum_nearest_psf_distance( 

439 image_mask, 

440 psf_cat, 

441 sampling=8, 

442 bad_mask_bits=["BAD", "CR", "INTRP", "SAT", "SUSPECT", "NO_DATA", "EDGE"], 

443): 

444 """Compute the maximum distance of an unmasked pixel to its nearest PSF. 

445 

446 Parameters 

447 ---------- 

448 image_mask : `lsst.afw.image.Mask` 

449 The mask plane associated with the exposure. 

450 psf_cat : `lsst.afw.table.SourceCatalog` or `astropy.table.Table` 

451 Catalog containing only the stars used in the PSF modeling. 

452 sampling : `int` 

453 Sampling rate in each dimension to create the grid of points on which 

454 to evaluate the distance to the nearest PSF star. The tradeoff is 

455 between adequate sampling versus speed. 

456 bad_mask_bits : `list` [`str`] 

457 Mask bits required to be absent for a pixel to be considered 

458 "unmasked". 

459 

460 Returns 

461 ------- 

462 max_dist_to_nearest_psf : `float` 

463 The maximum distance (in pixels) of an unmasked pixel to its nearest 

464 PSF model star. 

465 """ 

466 mask_arr = image_mask.array[::sampling, ::sampling] 

467 bitmask = image_mask.getPlaneBitMask(bad_mask_bits) 

468 good = ((mask_arr & bitmask) == 0) 

469 

470 x = np.arange(good.shape[1]) * sampling 

471 y = np.arange(good.shape[0]) * sampling 

472 xx, yy = np.meshgrid(x, y) 

473 

474 dist_to_nearest_psf = np.full(good.shape, np.inf) 

475 for psf in psf_cat: 

476 x_psf = psf["slot_Centroid_x"] 

477 y_psf = psf["slot_Centroid_y"] 

478 dist_to_nearest_psf = np.minimum(dist_to_nearest_psf, np.hypot(xx - x_psf, yy - y_psf)) 

479 unmasked_dists = dist_to_nearest_psf * good 

480 max_dist_to_nearest_psf = np.max(unmasked_dists) 

481 

482 return max_dist_to_nearest_psf 

483 

484 

485def psf_trace_radius_delta( 

486 image_mask, 

487 image_psf, 

488 sampling=96, 

489 bad_mask_bits=["BAD", "CR", "INTRP", "SAT", "SUSPECT", "NO_DATA", "EDGE"], 

490): 

491 """Compute the delta between the maximum and minimum model PSF trace radius 

492 values evaluated on a grid of points lying in the unmasked region of the 

493 image. 

494 

495 Parameters 

496 ---------- 

497 image_mask : `lsst.afw.image.Mask` 

498 The mask plane associated with the exposure. 

499 image_psf : `lsst.afw.detection.Psf` 

500 The PSF model associated with the exposure. 

501 sampling : `int` 

502 Sampling rate in each dimension to create the grid of points at which 

503 to evaluate ``image_psf``s trace radius value. The tradeoff is between 

504 adequate sampling versus speed. 

505 bad_mask_bits : `list` [`str`] 

506 Mask bits required to be absent for a pixel to be considered 

507 "unmasked". 

508 

509 Returns 

510 ------- 

511 psf_trace_radius_delta : `float` 

512 The delta (in pixels) between the maximum and minimum model PSF trace 

513 radius values evaluated on the x,y-grid subsampled on the unmasked 

514 detector pixels by a factor of ``sampling``. If any model PSF trace 

515 radius value on the grid evaluates to NaN, then NaN is returned 

516 immediately. 

517 """ 

518 psf_trace_radius_list = [] 

519 mask_arr = image_mask.array[::sampling, ::sampling] 

520 bitmask = image_mask.getPlaneBitMask(bad_mask_bits) 

521 good = ((mask_arr & bitmask) == 0) 

522 

523 x = np.arange(good.shape[1]) * sampling 

524 y = np.arange(good.shape[0]) * sampling 

525 xx, yy = np.meshgrid(x, y) 

526 

527 for x_mesh, y_mesh, good_mesh in zip(xx, yy, good): 

528 for x_point, y_point, is_good in zip(x_mesh, y_mesh, good_mesh): 

529 if is_good: 

530 psf_trace_radius = image_psf.computeShape(geom.Point2D(x_point, y_point)).getTraceRadius() 

531 if ~np.isfinite(psf_trace_radius): 

532 return float("nan") 

533 psf_trace_radius_list.append(psf_trace_radius) 

534 

535 psf_trace_radius_delta = np.max(psf_trace_radius_list) - np.min(psf_trace_radius_list) 

536 

537 return psf_trace_radius_delta