Coverage for python/lsst/pipe/tasks/computeExposureSummaryStats.py: 15%

198 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2024-03-14 10:10 +0000

1# This file is part of pipe_tasks. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21import warnings 

22import numpy as np 

23from scipy.stats import median_abs_deviation as sigmaMad 

24import astropy.units as units 

25from astropy.time import Time 

26from astropy.coordinates import AltAz, SkyCoord, EarthLocation 

27from lsst.daf.base import DateTime 

28 

29import lsst.pipe.base as pipeBase 

30import lsst.pex.config as pexConfig 

31import lsst.afw.math as afwMath 

32import lsst.afw.image as afwImage 

33import lsst.geom as geom 

34from lsst.utils.timer import timeMethod 

35 

36 

37__all__ = ("ComputeExposureSummaryStatsTask", "ComputeExposureSummaryStatsConfig") 

38 

39 

40class ComputeExposureSummaryStatsConfig(pexConfig.Config): 

41 """Config for ComputeExposureSummaryTask""" 

42 sigmaClip = pexConfig.Field( 

43 dtype=float, 

44 doc="Sigma for outlier rejection for sky noise.", 

45 default=3.0, 

46 ) 

47 clipIter = pexConfig.Field( 

48 dtype=int, 

49 doc="Number of iterations of outlier rejection for sky noise.", 

50 default=2, 

51 ) 

52 badMaskPlanes = pexConfig.ListField( 

53 dtype=str, 

54 doc="Mask planes that, if set, the associated pixel should not be included sky noise calculation.", 

55 default=("NO_DATA", "SUSPECT"), 

56 ) 

57 starSelection = pexConfig.Field( 

58 doc="Field to select sources to be used in the PSF statistics computation.", 

59 dtype=str, 

60 default="calib_psf_used" 

61 ) 

62 starShape = pexConfig.Field( 

63 doc="Base name of columns to use for the source shape in the PSF statistics computation.", 

64 dtype=str, 

65 default="slot_Shape" 

66 ) 

67 psfShape = pexConfig.Field( 

68 doc="Base name of columns to use for the PSF shape in the PSF statistics computation.", 

69 dtype=str, 

70 default="slot_PsfShape" 

71 ) 

72 psfSampling = pexConfig.Field( 

73 dtype=int, 

74 doc="Sampling rate in pixels in each dimension for the maxDistToNearestPsf metric " 

75 "caclulation grid (the tradeoff is between adequate sampling versus speed).", 

76 default=8, 

77 ) 

78 psfGridSampling = pexConfig.Field( 

79 dtype=int, 

80 doc="Sampling rate in pixels in each dimension for PSF model robustness metric " 

81 "caclulations grid (the tradeoff is between adequate sampling versus speed).", 

82 default=96, 

83 ) 

84 psfBadMaskPlanes = pexConfig.ListField( 

85 dtype=str, 

86 doc="Mask planes that, if set, the associated pixel should not be included in the PSF model " 

87 "robutsness metric calculations (namely, maxDistToNearestPsf and psfTraceRadiusDelta).", 

88 default=("BAD", "CR", "EDGE", "INTRP", "NO_DATA", "SAT", "SUSPECT"), 

89 ) 

90 

91 

92class ComputeExposureSummaryStatsTask(pipeBase.Task): 

93 """Task to compute exposure summary statistics. 

94 

95 This task computes various quantities suitable for DPDD and other 

96 downstream processing at the detector centers, including: 

97 - psfSigma 

98 - psfArea 

99 - psfIxx 

100 - psfIyy 

101 - psfIxy 

102 - ra 

103 - decl 

104 - zenithDistance 

105 - zeroPoint 

106 - skyBg 

107 - skyNoise 

108 - meanVar 

109 - raCorners 

110 - decCorners 

111 - astromOffsetMean 

112 - astromOffsetStd 

113 

114 These additional quantities are computed from the stars in the detector: 

115 - psfStarDeltaE1Median 

116 - psfStarDeltaE2Median 

117 - psfStarDeltaE1Scatter 

118 - psfStarDeltaE2Scatter 

119 - psfStarDeltaSizeMedian 

120 - psfStarDeltaSizeScatter 

121 - psfStarScaledDeltaSizeScatter 

122 

123 These quantities are computed based on the PSF model and image mask 

124 to assess the robustness of the PSF model across a given detector 

125 (against, e.g., extrapolation instability): 

126 - maxDistToNearestPsf 

127 - psfTraceRadiusDelta 

128 """ 

129 ConfigClass = ComputeExposureSummaryStatsConfig 

130 _DefaultName = "computeExposureSummaryStats" 

131 

132 @timeMethod 

133 def run(self, exposure, sources, background): 

134 """Measure exposure statistics from the exposure, sources, and 

135 background. 

136 

137 Parameters 

138 ---------- 

139 exposure : `lsst.afw.image.ExposureF` 

140 sources : `lsst.afw.table.SourceCatalog` 

141 background : `lsst.afw.math.BackgroundList` 

142 

143 Returns 

144 ------- 

145 summary : `lsst.afw.image.ExposureSummary` 

146 """ 

147 self.log.info("Measuring exposure statistics") 

148 

149 summary = afwImage.ExposureSummaryStats() 

150 

151 bbox = exposure.getBBox() 

152 

153 psf = exposure.getPsf() 

154 self.update_psf_stats(summary, psf, bbox, sources, image_mask=exposure.mask) 

155 

156 wcs = exposure.getWcs() 

157 visitInfo = exposure.getInfo().getVisitInfo() 

158 self.update_wcs_stats(summary, wcs, bbox, visitInfo) 

159 

160 photoCalib = exposure.getPhotoCalib() 

161 self.update_photo_calib_stats(summary, photoCalib) 

162 

163 self.update_background_stats(summary, background) 

164 

165 self.update_masked_image_stats(summary, exposure.getMaskedImage()) 

166 

167 md = exposure.getMetadata() 

168 if 'SFM_ASTROM_OFFSET_MEAN' in md: 

169 summary.astromOffsetMean = md['SFM_ASTROM_OFFSET_MEAN'] 

170 summary.astromOffsetStd = md['SFM_ASTROM_OFFSET_STD'] 

171 

172 return summary 

173 

174 def update_psf_stats(self, summary, psf, bbox, sources=None, image_mask=None, sources_is_astropy=False): 

175 """Compute all summary-statistic fields that depend on the PSF model. 

176 

177 Parameters 

178 ---------- 

179 summary : `lsst.afw.image.ExposureSummaryStats` 

180 Summary object to update in-place. 

181 psf : `lsst.afw.detection.Psf` or `None` 

182 Point spread function model. If `None`, all fields that depend on 

183 the PSF will be reset (generally to NaN). 

184 bbox : `lsst.geom.Box2I` 

185 Bounding box of the image for which summary stats are being 

186 computed. 

187 sources : `lsst.afw.table.SourceCatalog` or `astropy.table.Table` 

188 Catalog for quantities that are computed from source table columns. 

189 If `None`, these quantities will be reset (generally to NaN). 

190 The type of this table must correspond to the 

191 ``sources_is_astropy`` argument. 

192 image_mask : `lsst.afw.image.Mask`, optional 

193 Mask image that may be used to compute distance-to-nearest-star 

194 metrics. 

195 sources_is_astropy : `bool`, optional 

196 Whether ``sources`` is an `astropy.table.Table` instance instead 

197 of an `lsst.afw.table.Catalog` instance. Default is `False` (the 

198 latter). 

199 """ 

200 nan = float("nan") 

201 summary.psfSigma = nan 

202 summary.psfIxx = nan 

203 summary.psfIyy = nan 

204 summary.psfIxy = nan 

205 summary.psfArea = nan 

206 summary.nPsfStar = 0 

207 summary.psfStarDeltaE1Median = nan 

208 summary.psfStarDeltaE2Median = nan 

209 summary.psfStarDeltaE1Scatter = nan 

210 summary.psfStarDeltaE2Scatter = nan 

211 summary.psfStarDeltaSizeMedian = nan 

212 summary.psfStarDeltaSizeScatter = nan 

213 summary.psfStarScaledDeltaSizeScatter = nan 

214 summary.maxDistToNearestPsf = nan 

215 summary.psfTraceRadiusDelta = nan 

216 

217 if psf is None: 

218 return 

219 shape = psf.computeShape(bbox.getCenter()) 

220 summary.psfSigma = shape.getDeterminantRadius() 

221 summary.psfIxx = shape.getIxx() 

222 summary.psfIyy = shape.getIyy() 

223 summary.psfIxy = shape.getIxy() 

224 im = psf.computeKernelImage(bbox.getCenter()) 

225 # The calculation of effective psf area is taken from 

226 # meas_base/src/PsfFlux.cc#L112. See 

227 # https://github.com/lsst/meas_base/blob/ 

228 # 750bffe6620e565bda731add1509507f5c40c8bb/src/PsfFlux.cc#L112 

229 summary.psfArea = float(np.sum(im.array)/np.sum(im.array**2.)) 

230 

231 if image_mask is not None: 

232 psfTraceRadiusDelta = psf_trace_radius_delta( 

233 image_mask, 

234 psf, 

235 sampling=self.config.psfGridSampling, 

236 bad_mask_bits=self.config.psfBadMaskPlanes 

237 ) 

238 summary.psfTraceRadiusDelta = float(psfTraceRadiusDelta) 

239 

240 if sources is None: 

241 # No sources are available (as in some tests) 

242 return 

243 

244 psf_mask = sources[self.config.starSelection] & (~sources[self.config.starShape + '_flag']) 

245 nPsfStar = psf_mask.sum() 

246 

247 if nPsfStar == 0: 

248 # No stars to measure statistics, so we must return the defaults 

249 # of 0 stars and NaN values. 

250 return 

251 

252 if sources_is_astropy: 

253 psf_cat = sources[psf_mask] 

254 else: 

255 psf_cat = sources[psf_mask].copy(deep=True) 

256 

257 starXX = psf_cat[self.config.starShape + '_xx'] 

258 starYY = psf_cat[self.config.starShape + '_yy'] 

259 starXY = psf_cat[self.config.starShape + '_xy'] 

260 psfXX = psf_cat[self.config.psfShape + '_xx'] 

261 psfYY = psf_cat[self.config.psfShape + '_yy'] 

262 psfXY = psf_cat[self.config.psfShape + '_xy'] 

263 

264 starSize = (starXX*starYY - starXY**2.)**0.25 

265 starE1 = (starXX - starYY)/(starXX + starYY) 

266 starE2 = 2*starXY/(starXX + starYY) 

267 starSizeMedian = np.median(starSize) 

268 

269 psfSize = (psfXX*psfYY - psfXY**2)**0.25 

270 psfE1 = (psfXX - psfYY)/(psfXX + psfYY) 

271 psfE2 = 2*psfXY/(psfXX + psfYY) 

272 

273 psfStarDeltaE1Median = np.median(starE1 - psfE1) 

274 psfStarDeltaE1Scatter = sigmaMad(starE1 - psfE1, scale='normal') 

275 psfStarDeltaE2Median = np.median(starE2 - psfE2) 

276 psfStarDeltaE2Scatter = sigmaMad(starE2 - psfE2, scale='normal') 

277 

278 psfStarDeltaSizeMedian = np.median(starSize - psfSize) 

279 psfStarDeltaSizeScatter = sigmaMad(starSize - psfSize, scale='normal') 

280 psfStarScaledDeltaSizeScatter = psfStarDeltaSizeScatter/starSizeMedian**2. 

281 

282 summary.nPsfStar = int(nPsfStar) 

283 summary.psfStarDeltaE1Median = float(psfStarDeltaE1Median) 

284 summary.psfStarDeltaE2Median = float(psfStarDeltaE2Median) 

285 summary.psfStarDeltaE1Scatter = float(psfStarDeltaE1Scatter) 

286 summary.psfStarDeltaE2Scatter = float(psfStarDeltaE2Scatter) 

287 summary.psfStarDeltaSizeMedian = float(psfStarDeltaSizeMedian) 

288 summary.psfStarDeltaSizeScatter = float(psfStarDeltaSizeScatter) 

289 summary.psfStarScaledDeltaSizeScatter = float(psfStarScaledDeltaSizeScatter) 

290 

291 if image_mask is not None: 

292 maxDistToNearestPsf = maximum_nearest_psf_distance( 

293 image_mask, 

294 psf_cat, 

295 sampling=self.config.psfSampling, 

296 bad_mask_bits=self.config.psfBadMaskPlanes 

297 ) 

298 summary.maxDistToNearestPsf = float(maxDistToNearestPsf) 

299 

300 def update_wcs_stats(self, summary, wcs, bbox, visitInfo): 

301 """Compute all summary-statistic fields that depend on the WCS model. 

302 

303 Parameters 

304 ---------- 

305 summary : `lsst.afw.image.ExposureSummaryStats` 

306 Summary object to update in-place. 

307 wcs : `lsst.afw.geom.SkyWcs` or `None` 

308 Astrometric calibration model. If `None`, all fields that depend 

309 on the WCS will be reset (generally to NaN). 

310 bbox : `lsst.geom.Box2I` 

311 Bounding box of the image for which summary stats are being 

312 computed. 

313 visitInfo : `lsst.afw.image.VisitInfo` 

314 Observation information used in together with ``wcs`` to compute 

315 the zenith distance. 

316 """ 

317 nan = float("nan") 

318 summary.raCorners = [nan]*4 

319 summary.decCorners = [nan]*4 

320 summary.ra = nan 

321 summary.decl = nan 

322 summary.zenithDistance = nan 

323 

324 if wcs is None: 

325 return 

326 

327 sph_pts = wcs.pixelToSky(geom.Box2D(bbox).getCorners()) 

328 summary.raCorners = [float(sph.getRa().asDegrees()) for sph in sph_pts] 

329 summary.decCorners = [float(sph.getDec().asDegrees()) for sph in sph_pts] 

330 

331 sph_pt = wcs.pixelToSky(bbox.getCenter()) 

332 summary.ra = sph_pt.getRa().asDegrees() 

333 summary.decl = sph_pt.getDec().asDegrees() 

334 

335 date = visitInfo.getDate() 

336 

337 if date.isValid(): 

338 # We compute the zenithDistance at the center of the detector 

339 # rather than use the boresight value available via the visitInfo, 

340 # because the zenithDistance may vary significantly over a large 

341 # field of view. 

342 observatory = visitInfo.getObservatory() 

343 loc = EarthLocation(lat=observatory.getLatitude().asDegrees()*units.deg, 

344 lon=observatory.getLongitude().asDegrees()*units.deg, 

345 height=observatory.getElevation()*units.m) 

346 obstime = Time(visitInfo.getDate().get(system=DateTime.MJD), 

347 location=loc, format='mjd') 

348 coord = SkyCoord( 

349 summary.ra*units.degree, 

350 summary.decl*units.degree, 

351 obstime=obstime, 

352 location=loc, 

353 ) 

354 with warnings.catch_warnings(): 

355 warnings.simplefilter('ignore') 

356 altaz = coord.transform_to(AltAz) 

357 

358 summary.zenithDistance = float(90.0 - altaz.alt.degree) 

359 

360 def update_photo_calib_stats(self, summary, photo_calib): 

361 """Compute all summary-statistic fields that depend on the photometric 

362 calibration model. 

363 

364 Parameters 

365 ---------- 

366 summary : `lsst.afw.image.ExposureSummaryStats` 

367 Summary object to update in-place. 

368 photo_calib : `lsst.afw.image.PhotoCalib` or `None` 

369 Photometric calibration model. If `None`, all fields that depend 

370 on the photometric calibration will be reset (generally to NaN). 

371 """ 

372 if photo_calib is not None: 

373 summary.zeroPoint = float(2.5*np.log10(photo_calib.getInstFluxAtZeroMagnitude())) 

374 else: 

375 summary.zeroPoint = float("nan") 

376 

377 def update_background_stats(self, summary, background): 

378 """Compute summary-statistic fields that depend only on the 

379 background model. 

380 

381 Parameters 

382 ---------- 

383 summary : `lsst.afw.image.ExposureSummaryStats` 

384 Summary object to update in-place. 

385 background : `lsst.afw.math.BackgroundList` or `None` 

386 Background model. If `None`, all fields that depend on the 

387 background will be reset (generally to NaN). 

388 

389 Notes 

390 ----- 

391 This does not include fields that depend on the background-subtracted 

392 masked image; when the background changes, it should generally be 

393 applied to the image and `update_masked_image_stats` should be called 

394 as well. 

395 """ 

396 if background is not None: 

397 bgStats = (bg[0].getStatsImage().getImage().array 

398 for bg in background) 

399 summary.skyBg = float(sum(np.median(bg[np.isfinite(bg)]) for bg in bgStats)) 

400 else: 

401 summary.skyBg = float("nan") 

402 

403 def update_masked_image_stats(self, summary, masked_image): 

404 """Compute summary-statistic fields that depend on the masked image 

405 itself. 

406 

407 Parameters 

408 ---------- 

409 summary : `lsst.afw.image.ExposureSummaryStats` 

410 Summary object to update in-place. 

411 masked_image : `lsst.afw.image.MaskedImage` or `None` 

412 Masked image. If `None`, all fields that depend 

413 on the masked image will be reset (generally to NaN). 

414 """ 

415 nan = float("nan") 

416 if masked_image is None: 

417 summary.skyNoise = nan 

418 summary.meanVar = nan 

419 return 

420 statsCtrl = afwMath.StatisticsControl() 

421 statsCtrl.setNumSigmaClip(self.config.sigmaClip) 

422 statsCtrl.setNumIter(self.config.clipIter) 

423 statsCtrl.setAndMask(afwImage.Mask.getPlaneBitMask(self.config.badMaskPlanes)) 

424 statsCtrl.setNanSafe(True) 

425 

426 statObj = afwMath.makeStatistics(masked_image, afwMath.STDEVCLIP, statsCtrl) 

427 skyNoise, _ = statObj.getResult(afwMath.STDEVCLIP) 

428 summary.skyNoise = skyNoise 

429 

430 statObj = afwMath.makeStatistics(masked_image.variance, masked_image.mask, afwMath.MEANCLIP, 

431 statsCtrl) 

432 meanVar, _ = statObj.getResult(afwMath.MEANCLIP) 

433 summary.meanVar = meanVar 

434 

435 

436def maximum_nearest_psf_distance( 

437 image_mask, 

438 psf_cat, 

439 sampling=8, 

440 bad_mask_bits=["BAD", "CR", "INTRP", "SAT", "SUSPECT", "NO_DATA", "EDGE"], 

441): 

442 """Compute the maximum distance of an unmasked pixel to its nearest PSF. 

443 

444 Parameters 

445 ---------- 

446 image_mask : `lsst.afw.image.Mask` 

447 The mask plane associated with the exposure. 

448 psf_cat : `lsst.afw.table.SourceCatalog` or `astropy.table.Table` 

449 Catalog containing only the stars used in the PSF modeling. 

450 sampling : `int` 

451 Sampling rate in each dimension to create the grid of points on which 

452 to evaluate the distance to the nearest PSF star. The tradeoff is 

453 between adequate sampling versus speed. 

454 bad_mask_bits : `list` [`str`] 

455 Mask bits required to be absent for a pixel to be considered 

456 "unmasked". 

457 

458 Returns 

459 ------- 

460 max_dist_to_nearest_psf : `float` 

461 The maximum distance (in pixels) of an unmasked pixel to its nearest 

462 PSF model star. 

463 """ 

464 mask_arr = image_mask.array[::sampling, ::sampling] 

465 bitmask = image_mask.getPlaneBitMask(bad_mask_bits) 

466 good = ((mask_arr & bitmask) == 0) 

467 

468 x = np.arange(good.shape[1]) * sampling 

469 y = np.arange(good.shape[0]) * sampling 

470 xx, yy = np.meshgrid(x, y) 

471 

472 dist_to_nearest_psf = np.full(good.shape, np.inf) 

473 for psf in psf_cat: 

474 x_psf = psf["slot_Centroid_x"] 

475 y_psf = psf["slot_Centroid_y"] 

476 dist_to_nearest_psf = np.minimum(dist_to_nearest_psf, np.hypot(xx - x_psf, yy - y_psf)) 

477 unmasked_dists = dist_to_nearest_psf * good 

478 max_dist_to_nearest_psf = np.max(unmasked_dists) 

479 

480 return max_dist_to_nearest_psf 

481 

482 

483def psf_trace_radius_delta( 

484 image_mask, 

485 image_psf, 

486 sampling=96, 

487 bad_mask_bits=["BAD", "CR", "INTRP", "SAT", "SUSPECT", "NO_DATA", "EDGE"], 

488): 

489 """Compute the delta between the maximum and minimum model PSF trace radius 

490 values evaluated on a grid of points lying in the unmasked region of the 

491 image. 

492 

493 Parameters 

494 ---------- 

495 image_mask : `lsst.afw.image.Mask` 

496 The mask plane associated with the exposure. 

497 image_psf : `lsst.afw.detection.Psf` 

498 The PSF model associated with the exposure. 

499 sampling : `int` 

500 Sampling rate in each dimension to create the grid of points at which 

501 to evaluate ``image_psf``s trace radius value. The tradeoff is between 

502 adequate sampling versus speed. 

503 bad_mask_bits : `list` [`str`] 

504 Mask bits required to be absent for a pixel to be considered 

505 "unmasked". 

506 

507 Returns 

508 ------- 

509 psf_trace_radius_delta : `float` 

510 The delta (in pixels) between the maximum and minimum model PSF trace 

511 radius values evaluated on the x,y-grid subsampled on the unmasked 

512 detector pixels by a factor of ``sampling``. If any model PSF trace 

513 radius value on the grid evaluates to NaN, then NaN is returned 

514 immediately. 

515 """ 

516 psf_trace_radius_list = [] 

517 mask_arr = image_mask.array[::sampling, ::sampling] 

518 bitmask = image_mask.getPlaneBitMask(bad_mask_bits) 

519 good = ((mask_arr & bitmask) == 0) 

520 

521 x = np.arange(good.shape[1]) * sampling 

522 y = np.arange(good.shape[0]) * sampling 

523 xx, yy = np.meshgrid(x, y) 

524 

525 for x_mesh, y_mesh, good_mesh in zip(xx, yy, good): 

526 for x_point, y_point, is_good in zip(x_mesh, y_mesh, good_mesh): 

527 if is_good: 

528 psf_trace_radius = image_psf.computeShape(geom.Point2D(x_point, y_point)).getTraceRadius() 

529 if ~np.isfinite(psf_trace_radius): 

530 return float("nan") 

531 psf_trace_radius_list.append(psf_trace_radius) 

532 

533 psf_trace_radius_delta = np.max(psf_trace_radius_list) - np.min(psf_trace_radius_list) 

534 

535 return psf_trace_radius_delta