Coverage for python/lsst/meas/algorithms/measureApCorr.py: 15%

151 statements  

« prev     ^ index     » next       coverage.py v7.2.3, created at 2023-04-27 03:07 -0700

1# 

2# LSST Data Management System 

3# 

4# Copyright 2008-2017 AURA/LSST. 

5# 

6# This product includes software developed by the 

7# LSST Project (http://www.lsst.org/). 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <https://www.lsstcorp.org/LegalNotices/>. 

22# 

23 

24__all__ = ("MeasureApCorrConfig", "MeasureApCorrTask", "MeasureApCorrError") 

25 

26import numpy as np 

27from scipy.stats import median_abs_deviation 

28 

29import lsst.pex.config 

30from lsst.afw.image import ApCorrMap 

31from lsst.afw.math import ChebyshevBoundedField, ChebyshevBoundedFieldConfig 

32from lsst.pipe.base import Task, Struct 

33from lsst.meas.base.apCorrRegistry import getApCorrNameSet 

34 

35from .sourceSelector import sourceSelectorRegistry 

36 

37 

38class MeasureApCorrError(RuntimeError): 

39 pass 

40 

41 

42class _FluxNames: 

43 """A collection of flux-related names for a given flux measurement algorithm. 

44 

45 Parameters 

46 ---------- 

47 name : `str` 

48 Name of flux measurement algorithm, e.g. ``base_PsfFlux``. 

49 schema : `lsst.afw.table.Schema` 

50 Catalog schema containing the flux field. The ``{name}_instFlux``, 

51 ``{name}_instFluxErr``, ``{name}_flag`` fields are checked for 

52 existence, and the ``apcorr_{name}_used`` field is added. 

53 

54 Raises 

55 ------ 

56 KeyError if any of instFlux, instFluxErr, or flag fields is missing. 

57 """ 

58 def __init__(self, name, schema): 

59 self.fluxName = name + "_instFlux" 

60 if self.fluxName not in schema: 

61 raise KeyError("Could not find " + self.fluxName) 

62 self.errName = name + "_instFluxErr" 

63 if self.errName not in schema: 

64 raise KeyError("Could not find " + self.errName) 

65 self.flagName = name + "_flag" 

66 if self.flagName not in schema: 

67 raise KeyError("Cound not find " + self.flagName) 

68 self.usedName = "apcorr_" + name + "_used" 

69 schema.addField(self.usedName, type="Flag", 

70 doc="Set if source was used in measuring aperture correction.") 

71 

72 

73class MeasureApCorrConfig(lsst.pex.config.Config): 

74 """Configuration for MeasureApCorrTask. 

75 """ 

76 refFluxName = lsst.pex.config.Field( 

77 doc="Field name prefix for the flux other measurements should be aperture corrected to match", 

78 dtype=str, 

79 default="slot_CalibFlux", 

80 ) 

81 sourceSelector = sourceSelectorRegistry.makeField( 

82 doc="Selector that sets the stars that aperture corrections will be measured from.", 

83 default="science", 

84 ) 

85 minDegreesOfFreedom = lsst.pex.config.RangeField( 

86 doc="Minimum number of degrees of freedom (# of valid data points - # of parameters);" 

87 " if this is exceeded, the order of the fit is decreased (in both dimensions), and" 

88 " if we can't decrease it enough, we'll raise ValueError.", 

89 dtype=int, 

90 default=1, 

91 min=1, 

92 ) 

93 fitConfig = lsst.pex.config.ConfigField( 

94 doc="Configuration used in fitting the aperture correction fields.", 

95 dtype=ChebyshevBoundedFieldConfig, 

96 ) 

97 numIter = lsst.pex.config.Field( 

98 doc="Number of iterations for robust MAD sigma clipping.", 

99 dtype=int, 

100 default=4, 

101 ) 

102 numSigmaClip = lsst.pex.config.Field( 

103 doc="Number of robust MAD sigma to do clipping.", 

104 dtype=float, 

105 default=4.0, 

106 ) 

107 allowFailure = lsst.pex.config.ListField( 

108 doc="Allow these measurement algorithms to fail without an exception.", 

109 dtype=str, 

110 default=[], 

111 ) 

112 

113 def setDefaults(self): 

114 selector = self.sourceSelector["science"] 

115 

116 selector.doFluxLimit = False 

117 selector.doFlags = True 

118 selector.doUnresolved = True 

119 selector.doSignalToNoise = True 

120 selector.doIsolated = False 

121 selector.flags.good = [] 

122 selector.flags.bad = [ 

123 "base_PixelFlags_flag_edge", 

124 "base_PixelFlags_flag_interpolatedCenter", 

125 "base_PixelFlags_flag_saturatedCenter", 

126 "base_PixelFlags_flag_crCenter", 

127 "base_PixelFlags_flag_bad", 

128 "base_PixelFlags_flag_interpolated", 

129 "base_PixelFlags_flag_saturated", 

130 ] 

131 selector.signalToNoise.minimum = 200.0 

132 selector.signalToNoise.maximum = None 

133 selector.signalToNoise.fluxField = "base_PsfFlux_instFlux" 

134 selector.signalToNoise.errField = "base_PsfFlux_instFluxErr" 

135 

136 def validate(self): 

137 lsst.pex.config.Config.validate(self) 

138 if self.sourceSelector.target.usesMatches: 

139 raise lsst.pex.config.FieldValidationError( 

140 MeasureApCorrConfig.sourceSelector, 

141 self, 

142 "Star selectors that require matches are not permitted.") 

143 

144 

145class MeasureApCorrTask(Task): 

146 """Task to measure aperture correction. 

147 """ 

148 ConfigClass = MeasureApCorrConfig 

149 _DefaultName = "measureApCorr" 

150 

151 def __init__(self, schema, **kwargs): 

152 """Construct a MeasureApCorrTask 

153 

154 For every name in lsst.meas.base.getApCorrNameSet(): 

155 - If the corresponding flux fields exist in the schema: 

156 - Add a new field apcorr_{name}_used 

157 - Add an entry to the self.toCorrect dict 

158 - Otherwise silently skip the name 

159 """ 

160 Task.__init__(self, **kwargs) 

161 self.refFluxNames = _FluxNames(self.config.refFluxName, schema) 

162 self.toCorrect = {} # dict of flux field name prefix: FluxKeys instance 

163 for name in sorted(getApCorrNameSet()): 

164 try: 

165 self.toCorrect[name] = _FluxNames(name, schema) 

166 except KeyError: 

167 # if a field in the registry is missing, just ignore it. 

168 pass 

169 self.makeSubtask("sourceSelector") 

170 

171 def run(self, exposure, catalog): 

172 """Measure aperture correction 

173 

174 Parameters 

175 ---------- 

176 exposure : `lsst.afw.image.Exposure` 

177 Exposure aperture corrections are being measured on. The 

178 bounding box is retrieved from it, and it is passed to the 

179 sourceSelector. The output aperture correction map is *not* 

180 added to the exposure; this is left to the caller. 

181 catalog : `lsst.afw.table.SourceCatalog` 

182 SourceCatalog containing measurements to be used to 

183 compute aperture corrections. 

184 

185 Returns 

186 ------- 

187 Struct : `lsst.pipe.base.Struct` 

188 Contains the following: 

189 

190 ``apCorrMap`` 

191 aperture correction map (`lsst.afw.image.ApCorrMap`) 

192 that contains two entries for each flux field: 

193 - flux field (e.g. base_PsfFlux_instFlux): 2d model 

194 - flux sigma field (e.g. base_PsfFlux_instFluxErr): 2d model of error 

195 """ 

196 bbox = exposure.getBBox() 

197 import lsstDebug 

198 display = lsstDebug.Info(__name__).display 

199 doPause = lsstDebug.Info(__name__).doPause 

200 

201 self.log.info("Measuring aperture corrections for %d flux fields", len(self.toCorrect)) 

202 

203 # First, create a subset of the catalog that contains only selected stars 

204 # with non-flagged reference fluxes. 

205 selected = self.sourceSelector.run(catalog, exposure=exposure) 

206 

207 use = ( 

208 ~selected.sourceCat[self.refFluxNames.flagName] 

209 & (np.isfinite(selected.sourceCat[self.refFluxNames.fluxName])) 

210 ) 

211 goodRefCat = selected.sourceCat[use].copy() 

212 

213 apCorrMap = ApCorrMap() 

214 

215 # Outer loop over the fields we want to correct 

216 for name, fluxNames in self.toCorrect.items(): 

217 # Create a more restricted subset with only the objects where the to-be-correct flux 

218 # is not flagged. 

219 fluxes = goodRefCat[fluxNames.fluxName] 

220 with np.errstate(invalid="ignore"): # suppress NaN warnings. 

221 isGood = ( 

222 (~goodRefCat[fluxNames.flagName]) 

223 & (np.isfinite(fluxes)) 

224 & (fluxes > 0.0) 

225 ) 

226 

227 # The 1 is the minimum number of ctrl.computeSize() when the order 

228 # drops to 0 in both x and y. 

229 if (isGood.sum() - 1) < self.config.minDegreesOfFreedom: 

230 if name in self.config.allowFailure: 

231 self.log.warning("Unable to measure aperture correction for '%s': " 

232 "only %d sources, but require at least %d.", 

233 name, isGood.sum(), self.config.minDegreesOfFreedom + 1) 

234 continue 

235 else: 

236 msg = ("Unable to measure aperture correction for required algorithm '%s': " 

237 "only %d sources, but require at least %d." % 

238 (name, isGood.sum(), self.config.minDegreesOfFreedom + 1)) 

239 self.log.warning(msg) 

240 raise MeasureApCorrError("Aperture correction failed on required algorithm.") 

241 

242 goodCat = goodRefCat[isGood].copy() 

243 

244 x = goodCat['slot_Centroid_x'] 

245 y = goodCat['slot_Centroid_y'] 

246 z = goodCat[self.refFluxNames.fluxName]/goodCat[fluxNames.fluxName] 

247 ids = goodCat['id'] 

248 

249 # We start with an initial fit that is the median offset; this 

250 # works well in practice. 

251 fitValues = np.median(z) 

252 

253 ctrl = self.config.fitConfig.makeControl() 

254 

255 allBad = False 

256 for iteration in range(self.config.numIter): 

257 resid = z - fitValues 

258 # We add a small (epsilon) amount of floating-point slop because 

259 # the median_abs_deviation may give a value that is just larger than 0 

260 # even if given a completely flat residual field (as in tests). 

261 apCorrErr = median_abs_deviation(resid, scale="normal") + 1e-7 

262 keep = np.abs(resid) <= self.config.numSigmaClip * apCorrErr 

263 

264 self.log.debug("Removing %d sources as outliers.", len(resid) - keep.sum()) 

265 

266 x = x[keep] 

267 y = y[keep] 

268 z = z[keep] 

269 ids = ids[keep] 

270 

271 while (len(x) - ctrl.computeSize()) < self.config.minDegreesOfFreedom: 

272 if ctrl.orderX > 0: 

273 ctrl.orderX -= 1 

274 else: 

275 allBad = True 

276 break 

277 if ctrl.orderY > 0: 

278 ctrl.orderY -= 1 

279 else: 

280 allBad = True 

281 break 

282 

283 if allBad: 

284 if name in self.config.allowFailure: 

285 self.log.warning("Unable to measure aperture correction for '%s': " 

286 "only %d sources remain, but require at least %d." % 

287 (name, keep.sum(), self.config.minDegreesOfFreedom + 1)) 

288 break 

289 else: 

290 raise RuntimeError("Unable to measure aperture correction for required algorithm " 

291 "'%s': only %d sources remain, but require at least %d." % 

292 (name, keep.sum(), self.config.minDegreesOfFreedom + 1)) 

293 

294 apCorrField = ChebyshevBoundedField.fit(bbox, x, y, z, ctrl) 

295 fitValues = apCorrField.evaluate(x, y) 

296 

297 if allBad: 

298 continue 

299 

300 self.log.info( 

301 "Aperture correction for %s from %d stars: MAD %f, RMS %f", 

302 name, 

303 median_abs_deviation(fitValues - z, scale="normal"), 

304 np.mean((fitValues - z)**2.)**0.5, 

305 len(x), 

306 ) 

307 

308 if display: 

309 plotApCorr(bbox, x, y, z, apCorrField, "%s, final" % (name,), doPause) 

310 

311 # Record which sources were used. 

312 used = np.zeros(len(catalog), dtype=bool) 

313 used[np.searchsorted(catalog['id'], ids)] = True 

314 catalog[fluxNames.usedName] = used 

315 

316 # Save the result in the output map 

317 # The error is constant spatially (we could imagine being 

318 # more clever, but we're not yet sure if it's worth the effort). 

319 # We save the errors as a 0th-order ChebyshevBoundedField 

320 apCorrMap[fluxNames.fluxName] = apCorrField 

321 apCorrMap[fluxNames.errName] = ChebyshevBoundedField( 

322 bbox, 

323 np.array([[apCorrErr]]), 

324 ) 

325 

326 return Struct( 

327 apCorrMap=apCorrMap, 

328 ) 

329 

330 

331def plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause): 

332 """Plot aperture correction fit residuals 

333 

334 There are two subplots: residuals against x and y. 

335 

336 Intended for debugging. 

337 

338 Parameters 

339 ---------- 

340 bbox : `lsst.geom.Box2I` 

341 Bounding box (for bounds) 

342 xx, yy : `numpy.ndarray`, (N) 

343 x and y coordinates 

344 zzMeasure : `float` 

345 Measured value of the aperture correction 

346 field : `lsst.afw.math.ChebyshevBoundedField` 

347 Fit aperture correction field 

348 title : 'str' 

349 Title for plot 

350 doPause : `bool` 

351 Pause to inspect the residuals plot? If 

352 False, there will be a 4 second delay to 

353 allow for inspection of the plot before 

354 closing it and moving on. 

355 """ 

356 import matplotlib.pyplot as plt 

357 

358 zzFit = field.evaluate(xx, yy) 

359 residuals = zzMeasure - zzFit 

360 

361 fig, axes = plt.subplots(2, 1) 

362 

363 axes[0].scatter(xx, residuals, s=3, marker='o', lw=0, alpha=0.7) 

364 axes[1].scatter(yy, residuals, s=3, marker='o', lw=0, alpha=0.7) 

365 for ax in axes: 

366 ax.set_ylabel("ApCorr Fit Residual") 

367 ax.set_ylim(0.9*residuals.min(), 1.1*residuals.max()) 

368 axes[0].set_xlabel("x") 

369 axes[0].set_xlim(bbox.getMinX(), bbox.getMaxX()) 

370 axes[1].set_xlabel("y") 

371 axes[1].set_xlim(bbox.getMinY(), bbox.getMaxY()) 

372 plt.suptitle(title) 

373 

374 if not doPause: 

375 try: 

376 plt.pause(4) 

377 plt.close() 

378 except Exception: 

379 print("%s: plt.pause() failed. Please close plots when done." % __name__) 

380 plt.show() 

381 else: 

382 print("%s: Please close plots when done." % __name__) 

383 plt.show()