Coverage for python/lsst/meas/algorithms/measureApCorr.py: 15%

147 statements  

« prev     ^ index     » next       coverage.py v6.5.0, created at 2023-03-16 02:19 -0700

1# 

2# LSST Data Management System 

3# 

4# Copyright 2008-2017 AURA/LSST. 

5# 

6# This product includes software developed by the 

7# LSST Project (http://www.lsst.org/). 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <https://www.lsstcorp.org/LegalNotices/>. 

22# 

23 

24__all__ = ("MeasureApCorrConfig", "MeasureApCorrTask") 

25 

26import numpy as np 

27from scipy.stats import median_abs_deviation 

28 

29import lsst.pex.config 

30from lsst.afw.image import ApCorrMap 

31from lsst.afw.math import ChebyshevBoundedField, ChebyshevBoundedFieldConfig 

32from lsst.pipe.base import Task, Struct 

33from lsst.meas.base.apCorrRegistry import getApCorrNameSet 

34 

35from .sourceSelector import sourceSelectorRegistry 

36 

37 

38class _FluxNames: 

39 """A collection of flux-related names for a given flux measurement algorithm. 

40 

41 Parameters 

42 ---------- 

43 name : `str` 

44 Name of flux measurement algorithm, e.g. ``base_PsfFlux``. 

45 schema : `lsst.afw.table.Schema` 

46 Catalog schema containing the flux field. The ``{name}_instFlux``, 

47 ``{name}_instFluxErr``, ``{name}_flag`` fields are checked for 

48 existence, and the ``apcorr_{name}_used`` field is added. 

49 

50 Raises 

51 ------ 

52 KeyError if any of instFlux, instFluxErr, or flag fields is missing. 

53 """ 

54 def __init__(self, name, schema): 

55 self.fluxName = name + "_instFlux" 

56 if self.fluxName not in schema: 

57 raise KeyError("Could not find " + self.fluxName) 

58 self.errName = name + "_instFluxErr" 

59 if self.errName not in schema: 

60 raise KeyError("Could not find " + self.errName) 

61 self.flagName = name + "_flag" 

62 if self.flagName not in schema: 

63 raise KeyError("Cound not find " + self.flagName) 

64 self.usedName = "apcorr_" + name + "_used" 

65 schema.addField(self.usedName, type="Flag", 

66 doc="Set if source was used in measuring aperture correction.") 

67 

68 

69class MeasureApCorrConfig(lsst.pex.config.Config): 

70 """Configuration for MeasureApCorrTask. 

71 """ 

72 refFluxName = lsst.pex.config.Field( 

73 doc="Field name prefix for the flux other measurements should be aperture corrected to match", 

74 dtype=str, 

75 default="slot_CalibFlux", 

76 ) 

77 sourceSelector = sourceSelectorRegistry.makeField( 

78 doc="Selector that sets the stars that aperture corrections will be measured from.", 

79 default="science", 

80 ) 

81 minDegreesOfFreedom = lsst.pex.config.RangeField( 

82 doc="Minimum number of degrees of freedom (# of valid data points - # of parameters);" 

83 " if this is exceeded, the order of the fit is decreased (in both dimensions), and" 

84 " if we can't decrease it enough, we'll raise ValueError.", 

85 dtype=int, 

86 default=1, 

87 min=1, 

88 ) 

89 fitConfig = lsst.pex.config.ConfigField( 

90 doc="Configuration used in fitting the aperture correction fields.", 

91 dtype=ChebyshevBoundedFieldConfig, 

92 ) 

93 numIter = lsst.pex.config.Field( 

94 doc="Number of iterations for robust MAD sigma clipping.", 

95 dtype=int, 

96 default=4, 

97 ) 

98 numSigmaClip = lsst.pex.config.Field( 

99 doc="Number of robust MAD sigma to do clipping.", 

100 dtype=float, 

101 default=4.0, 

102 ) 

103 allowFailure = lsst.pex.config.ListField( 

104 doc="Allow these measurement algorithms to fail without an exception.", 

105 dtype=str, 

106 default=[], 

107 ) 

108 

109 def setDefaults(self): 

110 selector = self.sourceSelector["science"] 

111 

112 selector.doFluxLimit = False 

113 selector.doFlags = True 

114 selector.doUnresolved = True 

115 selector.doSignalToNoise = True 

116 selector.doIsolated = False 

117 selector.flags.good = [] 

118 selector.flags.bad = [ 

119 "base_PixelFlags_flag_edge", 

120 "base_PixelFlags_flag_interpolatedCenter", 

121 "base_PixelFlags_flag_saturatedCenter", 

122 "base_PixelFlags_flag_crCenter", 

123 "base_PixelFlags_flag_bad", 

124 "base_PixelFlags_flag_interpolated", 

125 "base_PixelFlags_flag_saturated", 

126 ] 

127 selector.signalToNoise.minimum = 200.0 

128 selector.signalToNoise.maximum = None 

129 selector.signalToNoise.fluxField = "base_PsfFlux_instFlux" 

130 selector.signalToNoise.errField = "base_PsfFlux_instFluxErr" 

131 

132 def validate(self): 

133 lsst.pex.config.Config.validate(self) 

134 if self.sourceSelector.target.usesMatches: 

135 raise lsst.pex.config.FieldValidationError( 

136 MeasureApCorrConfig.sourceSelector, 

137 self, 

138 "Star selectors that require matches are not permitted.") 

139 

140 

141class MeasureApCorrTask(Task): 

142 """Task to measure aperture correction. 

143 """ 

144 ConfigClass = MeasureApCorrConfig 

145 _DefaultName = "measureApCorr" 

146 

147 def __init__(self, schema, **kwargs): 

148 """Construct a MeasureApCorrTask 

149 

150 For every name in lsst.meas.base.getApCorrNameSet(): 

151 - If the corresponding flux fields exist in the schema: 

152 - Add a new field apcorr_{name}_used 

153 - Add an entry to the self.toCorrect dict 

154 - Otherwise silently skip the name 

155 """ 

156 Task.__init__(self, **kwargs) 

157 self.refFluxNames = _FluxNames(self.config.refFluxName, schema) 

158 self.toCorrect = {} # dict of flux field name prefix: FluxKeys instance 

159 for name in sorted(getApCorrNameSet()): 

160 try: 

161 self.toCorrect[name] = _FluxNames(name, schema) 

162 except KeyError: 

163 # if a field in the registry is missing, just ignore it. 

164 pass 

165 self.makeSubtask("sourceSelector") 

166 

167 def run(self, exposure, catalog): 

168 """Measure aperture correction 

169 

170 Parameters 

171 ---------- 

172 exposure : `lsst.afw.image.Exposure` 

173 Exposure aperture corrections are being measured on. The 

174 bounding box is retrieved from it, and it is passed to the 

175 sourceSelector. The output aperture correction map is *not* 

176 added to the exposure; this is left to the caller. 

177 catalog : `lsst.afw.table.SourceCatalog` 

178 SourceCatalog containing measurements to be used to 

179 compute aperture corrections. 

180 

181 Returns 

182 ------- 

183 Struct : `lsst.pipe.base.Struct` 

184 Contains the following: 

185 

186 ``apCorrMap`` 

187 aperture correction map (`lsst.afw.image.ApCorrMap`) 

188 that contains two entries for each flux field: 

189 - flux field (e.g. base_PsfFlux_instFlux): 2d model 

190 - flux sigma field (e.g. base_PsfFlux_instFluxErr): 2d model of error 

191 """ 

192 bbox = exposure.getBBox() 

193 import lsstDebug 

194 display = lsstDebug.Info(__name__).display 

195 doPause = lsstDebug.Info(__name__).doPause 

196 

197 self.log.info("Measuring aperture corrections for %d flux fields", len(self.toCorrect)) 

198 

199 # First, create a subset of the catalog that contains only selected stars 

200 # with non-flagged reference fluxes. 

201 selected = self.sourceSelector.run(catalog, exposure=exposure) 

202 

203 use = ( 

204 ~selected.sourceCat[self.refFluxNames.flagName] 

205 & (np.isfinite(selected.sourceCat[self.refFluxNames.fluxName])) 

206 ) 

207 goodRefCat = selected.sourceCat[use].copy() 

208 

209 apCorrMap = ApCorrMap() 

210 

211 # Outer loop over the fields we want to correct 

212 for name, fluxNames in self.toCorrect.items(): 

213 # Create a more restricted subset with only the objects where the to-be-correct flux 

214 # is not flagged. 

215 fluxes = goodRefCat[fluxNames.fluxName] 

216 with np.errstate(invalid="ignore"): # suppress NaN warnings. 

217 isGood = ( 

218 (~goodRefCat[fluxNames.flagName]) 

219 & (np.isfinite(fluxes)) 

220 & (fluxes > 0.0) 

221 ) 

222 

223 # The 1 is the minimum number of ctrl.computeSize() when the order 

224 # drops to 0 in both x and y. 

225 if (isGood.sum() - 1) < self.config.minDegreesOfFreedom: 

226 if name in self.config.allowFailure: 

227 self.log.warning("Unable to measure aperture correction for '%s': " 

228 "only %d sources, but require at least %d." % 

229 (name, isGood.sum(), self.config.minDegreesOfFreedom + 1)) 

230 continue 

231 else: 

232 raise RuntimeError("Unable to measure aperture correction for required algorithm '%s': " 

233 "only %d sources, but require at least %d." % 

234 (name, isGood.sum(), self.config.minDegreesOfFreedom + 1)) 

235 

236 goodCat = goodRefCat[isGood].copy() 

237 

238 x = goodCat['slot_Centroid_x'] 

239 y = goodCat['slot_Centroid_y'] 

240 z = goodCat[self.refFluxNames.fluxName]/goodCat[fluxNames.fluxName] 

241 ids = goodCat['id'] 

242 

243 # We start with an initial fit that is the median offset; this 

244 # works well in practice. 

245 fitValues = np.median(z) 

246 

247 ctrl = self.config.fitConfig.makeControl() 

248 

249 allBad = False 

250 for iteration in range(self.config.numIter): 

251 resid = z - fitValues 

252 # We add a small (epsilon) amount of floating-point slop because 

253 # the median_abs_deviation may give a value that is just larger than 0 

254 # even if given a completely flat residual field (as in tests). 

255 apCorrErr = median_abs_deviation(resid, scale="normal") + 1e-7 

256 keep = np.abs(resid) <= self.config.numSigmaClip * apCorrErr 

257 

258 self.log.debug("Removing %d sources as outliers.", len(resid) - keep.sum()) 

259 

260 x = x[keep] 

261 y = y[keep] 

262 z = z[keep] 

263 ids = ids[keep] 

264 

265 while (len(x) - ctrl.computeSize()) < self.config.minDegreesOfFreedom: 

266 if ctrl.orderX > 0: 

267 ctrl.orderX -= 1 

268 else: 

269 allBad = True 

270 break 

271 if ctrl.orderY > 0: 

272 ctrl.orderY -= 1 

273 else: 

274 allBad = True 

275 break 

276 

277 if allBad: 

278 if name in self.config.allowFailure: 

279 self.log.warning("Unable to measure aperture correction for '%s': " 

280 "only %d sources remain, but require at least %d." % 

281 (name, keep.sum(), self.config.minDegreesOfFreedom + 1)) 

282 break 

283 else: 

284 raise RuntimeError("Unable to measure aperture correction for required algorithm " 

285 "'%s': only %d sources remain, but require at least %d." % 

286 (name, keep.sum(), self.config.minDegreesOfFreedom + 1)) 

287 

288 apCorrField = ChebyshevBoundedField.fit(bbox, x, y, z, ctrl) 

289 fitValues = apCorrField.evaluate(x, y) 

290 

291 if allBad: 

292 continue 

293 

294 self.log.info( 

295 "Aperture correction for %s from %d stars: MAD %f, RMS %f", 

296 name, 

297 median_abs_deviation(fitValues - z, scale="normal"), 

298 np.mean((fitValues - z)**2.)**0.5, 

299 len(x), 

300 ) 

301 

302 if display: 

303 plotApCorr(bbox, x, y, z, apCorrField, "%s, final" % (name,), doPause) 

304 

305 # Record which sources were used. 

306 used = np.zeros(len(catalog), dtype=bool) 

307 used[np.searchsorted(catalog['id'], ids)] = True 

308 catalog[fluxNames.usedName] = used 

309 

310 # Save the result in the output map 

311 # The error is constant spatially (we could imagine being 

312 # more clever, but we're not yet sure if it's worth the effort). 

313 # We save the errors as a 0th-order ChebyshevBoundedField 

314 apCorrMap[fluxNames.fluxName] = apCorrField 

315 apCorrMap[fluxNames.errName] = ChebyshevBoundedField( 

316 bbox, 

317 np.array([[apCorrErr]]), 

318 ) 

319 

320 return Struct( 

321 apCorrMap=apCorrMap, 

322 ) 

323 

324 

325def plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause): 

326 """Plot aperture correction fit residuals 

327 

328 There are two subplots: residuals against x and y. 

329 

330 Intended for debugging. 

331 

332 Parameters 

333 ---------- 

334 bbox : `lsst.geom.Box2I` 

335 Bounding box (for bounds) 

336 xx, yy : `numpy.ndarray`, (N) 

337 x and y coordinates 

338 zzMeasure : `float` 

339 Measured value of the aperture correction 

340 field : `lsst.afw.math.ChebyshevBoundedField` 

341 Fit aperture correction field 

342 title : 'str' 

343 Title for plot 

344 doPause : `bool` 

345 Pause to inspect the residuals plot? If 

346 False, there will be a 4 second delay to 

347 allow for inspection of the plot before 

348 closing it and moving on. 

349 """ 

350 import matplotlib.pyplot as plt 

351 

352 zzFit = field.evaluate(xx, yy) 

353 residuals = zzMeasure - zzFit 

354 

355 fig, axes = plt.subplots(2, 1) 

356 

357 axes[0].scatter(xx, residuals, s=3, marker='o', lw=0, alpha=0.7) 

358 axes[1].scatter(yy, residuals, s=3, marker='o', lw=0, alpha=0.7) 

359 for ax in axes: 

360 ax.set_ylabel("ApCorr Fit Residual") 

361 ax.set_ylim(0.9*residuals.min(), 1.1*residuals.max()) 

362 axes[0].set_xlabel("x") 

363 axes[0].set_xlim(bbox.getMinX(), bbox.getMaxX()) 

364 axes[1].set_xlabel("y") 

365 axes[1].set_xlim(bbox.getMinY(), bbox.getMaxY()) 

366 plt.suptitle(title) 

367 

368 if not doPause: 

369 try: 

370 plt.pause(4) 

371 plt.close() 

372 except Exception: 

373 print("%s: plt.pause() failed. Please close plots when done." % __name__) 

374 plt.show() 

375 else: 

376 print("%s: Please close plots when done." % __name__) 

377 plt.show()