Coverage for python/lsst/meas/algorithms/measureApCorr.py: 15%

153 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-06-17 09:33 +0000

1# 

2# LSST Data Management System 

3# 

4# Copyright 2008-2017 AURA/LSST. 

5# 

6# This product includes software developed by the 

7# LSST Project (http://www.lsst.org/). 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <https://www.lsstcorp.org/LegalNotices/>. 

22# 

23 

24__all__ = ("MeasureApCorrConfig", "MeasureApCorrTask", "MeasureApCorrError") 

25 

26import numpy as np 

27from scipy.stats import median_abs_deviation 

28 

29import lsst.pex.config 

30from lsst.afw.image import ApCorrMap 

31from lsst.afw.math import ChebyshevBoundedField, ChebyshevBoundedFieldConfig 

32from lsst.pipe.base import Task, Struct 

33from lsst.meas.base.apCorrRegistry import getApCorrNameSet 

34 

35from .sourceSelector import sourceSelectorRegistry 

36 

37 

38class MeasureApCorrError(RuntimeError): 

39 pass 

40 

41 

42class _FluxNames: 

43 """A collection of flux-related names for a given flux measurement algorithm. 

44 

45 Parameters 

46 ---------- 

47 name : `str` 

48 Name of flux measurement algorithm, e.g. ``base_PsfFlux``. 

49 schema : `lsst.afw.table.Schema` 

50 Catalog schema containing the flux field. The ``{name}_instFlux``, 

51 ``{name}_instFluxErr``, ``{name}_flag`` fields are checked for 

52 existence, and the ``apcorr_{name}_used`` field is added. 

53 

54 Raises 

55 ------ 

56 KeyError if any of instFlux, instFluxErr, or flag fields is missing. 

57 """ 

58 def __init__(self, name, schema): 

59 self.fluxName = name + "_instFlux" 

60 if self.fluxName not in schema: 

61 raise KeyError("Could not find " + self.fluxName) 

62 self.errName = name + "_instFluxErr" 

63 if self.errName not in schema: 

64 raise KeyError("Could not find " + self.errName) 

65 self.flagName = name + "_flag" 

66 if self.flagName not in schema: 

67 raise KeyError("Cound not find " + self.flagName) 

68 self.usedName = "apcorr_" + name + "_used" 

69 schema.addField(self.usedName, type="Flag", 

70 doc="Set if source was used in measuring aperture correction.") 

71 

72 

73class MeasureApCorrConfig(lsst.pex.config.Config): 

74 """Configuration for MeasureApCorrTask. 

75 """ 

76 refFluxName = lsst.pex.config.Field( 

77 doc="Field name prefix for the flux other measurements should be aperture corrected to match", 

78 dtype=str, 

79 default="slot_CalibFlux", 

80 ) 

81 sourceSelector = sourceSelectorRegistry.makeField( 

82 doc="Selector that sets the stars that aperture corrections will be measured from.", 

83 default="science", 

84 ) 

85 minDegreesOfFreedom = lsst.pex.config.RangeField( 

86 doc="Minimum number of degrees of freedom (# of valid data points - # of parameters);" 

87 " if this is exceeded, the order of the fit is decreased (in both dimensions), and" 

88 " if we can't decrease it enough, we'll raise ValueError.", 

89 dtype=int, 

90 default=1, 

91 min=1, 

92 ) 

93 fitConfig = lsst.pex.config.ConfigField( 

94 doc="Configuration used in fitting the aperture correction fields.", 

95 dtype=ChebyshevBoundedFieldConfig, 

96 ) 

97 numIter = lsst.pex.config.Field( 

98 doc="Number of iterations for robust MAD sigma clipping.", 

99 dtype=int, 

100 default=4, 

101 ) 

102 numSigmaClip = lsst.pex.config.Field( 

103 doc="Number of robust MAD sigma to do clipping.", 

104 dtype=float, 

105 default=4.0, 

106 ) 

107 allowFailure = lsst.pex.config.ListField( 

108 doc="Allow these measurement algorithms to fail without an exception.", 

109 dtype=str, 

110 default=[], 

111 ) 

112 

113 def setDefaults(self): 

114 selector = self.sourceSelector["science"] 

115 

116 selector.doFluxLimit = False 

117 selector.doFlags = True 

118 selector.doUnresolved = True 

119 selector.doSignalToNoise = True 

120 selector.doIsolated = False 

121 selector.flags.good = [] 

122 selector.flags.bad = [ 

123 "base_PixelFlags_flag_edge", 

124 "base_PixelFlags_flag_interpolatedCenter", 

125 "base_PixelFlags_flag_saturatedCenter", 

126 "base_PixelFlags_flag_crCenter", 

127 "base_PixelFlags_flag_bad", 

128 "base_PixelFlags_flag_interpolated", 

129 "base_PixelFlags_flag_saturated", 

130 ] 

131 selector.signalToNoise.minimum = 200.0 

132 selector.signalToNoise.maximum = None 

133 selector.signalToNoise.fluxField = "base_PsfFlux_instFlux" 

134 selector.signalToNoise.errField = "base_PsfFlux_instFluxErr" 

135 

136 def validate(self): 

137 lsst.pex.config.Config.validate(self) 

138 if self.sourceSelector.target.usesMatches: 

139 raise lsst.pex.config.FieldValidationError( 

140 MeasureApCorrConfig.sourceSelector, 

141 self, 

142 "Star selectors that require matches are not permitted.") 

143 

144 

145class MeasureApCorrTask(Task): 

146 """Task to measure aperture correction. 

147 """ 

148 ConfigClass = MeasureApCorrConfig 

149 _DefaultName = "measureApCorr" 

150 

151 def __init__(self, schema, **kwargs): 

152 """Construct a MeasureApCorrTask 

153 

154 For every name in lsst.meas.base.getApCorrNameSet(): 

155 - If the corresponding flux fields exist in the schema: 

156 - Add a new field apcorr_{name}_used 

157 - Add an entry to the self.toCorrect dict 

158 - Otherwise silently skip the name 

159 """ 

160 Task.__init__(self, **kwargs) 

161 self.refFluxNames = _FluxNames(self.config.refFluxName, schema) 

162 self.toCorrect = {} # dict of flux field name prefix: FluxKeys instance 

163 for name in sorted(getApCorrNameSet()): 

164 try: 

165 self.toCorrect[name] = _FluxNames(name, schema) 

166 except KeyError: 

167 # if a field in the registry is missing, just ignore it. 

168 pass 

169 self.makeSubtask("sourceSelector") 

170 

171 def run(self, exposure, catalog): 

172 """Measure aperture correction 

173 

174 Parameters 

175 ---------- 

176 exposure : `lsst.afw.image.Exposure` 

177 Exposure aperture corrections are being measured on. The 

178 bounding box is retrieved from it, and it is passed to the 

179 sourceSelector. The output aperture correction map is *not* 

180 added to the exposure; this is left to the caller. 

181 catalog : `lsst.afw.table.SourceCatalog` 

182 SourceCatalog containing measurements to be used to 

183 compute aperture corrections. 

184 

185 Returns 

186 ------- 

187 Struct : `lsst.pipe.base.Struct` 

188 Contains the following: 

189 

190 ``apCorrMap`` 

191 aperture correction map (`lsst.afw.image.ApCorrMap`) 

192 that contains two entries for each flux field: 

193 - flux field (e.g. base_PsfFlux_instFlux): 2d model 

194 - flux sigma field (e.g. base_PsfFlux_instFluxErr): 2d model of error 

195 """ 

196 bbox = exposure.getBBox() 

197 import lsstDebug 

198 display = lsstDebug.Info(__name__).display 

199 doPause = lsstDebug.Info(__name__).doPause 

200 

201 self.log.info("Measuring aperture corrections for %d flux fields", len(self.toCorrect)) 

202 

203 # First, create a subset of the catalog that contains only selected stars 

204 # with non-flagged reference fluxes. 

205 selected = self.sourceSelector.run(catalog, exposure=exposure) 

206 

207 use = ( 

208 ~selected.sourceCat[self.refFluxNames.flagName] 

209 & (np.isfinite(selected.sourceCat[self.refFluxNames.fluxName])) 

210 ) 

211 goodRefCat = selected.sourceCat[use].copy() 

212 

213 apCorrMap = ApCorrMap() 

214 

215 # Outer loop over the fields we want to correct 

216 for name, fluxNames in self.toCorrect.items(): 

217 # Create a more restricted subset with only the objects where the to-be-correct flux 

218 # is not flagged. 

219 fluxes = goodRefCat[fluxNames.fluxName] 

220 with np.errstate(invalid="ignore"): # suppress NaN warnings. 

221 isGood = ( 

222 (~goodRefCat[fluxNames.flagName]) 

223 & (np.isfinite(fluxes)) 

224 & (fluxes > 0.0) 

225 ) 

226 

227 # The 1 is the minimum number of ctrl.computeSize() when the order 

228 # drops to 0 in both x and y. 

229 if (isGood.sum() - 1) < self.config.minDegreesOfFreedom: 

230 if name in self.config.allowFailure: 

231 self.log.warning("Unable to measure aperture correction for '%s': " 

232 "only %d sources, but require at least %d.", 

233 name, isGood.sum(), self.config.minDegreesOfFreedom + 1) 

234 continue 

235 else: 

236 msg = ("Unable to measure aperture correction for required algorithm '%s': " 

237 "only %d sources, but require at least %d." % 

238 (name, isGood.sum(), self.config.minDegreesOfFreedom + 1)) 

239 self.log.warning(msg) 

240 raise MeasureApCorrError("Aperture correction failed on required algorithm.") 

241 

242 goodCat = goodRefCat[isGood].copy() 

243 

244 x = goodCat['slot_Centroid_x'] 

245 y = goodCat['slot_Centroid_y'] 

246 z = goodCat[self.refFluxNames.fluxName]/goodCat[fluxNames.fluxName] 

247 ids = goodCat['id'] 

248 

249 # We start with an initial fit that is the median offset; this 

250 # works well in practice. 

251 fitValues = np.median(z) 

252 

253 ctrl = self.config.fitConfig.makeControl() 

254 

255 allBad = False 

256 for iteration in range(self.config.numIter): 

257 resid = z - fitValues 

258 # We add a small (epsilon) amount of floating-point slop because 

259 # the median_abs_deviation may give a value that is just larger than 0 

260 # even if given a completely flat residual field (as in tests). 

261 apCorrErr = median_abs_deviation(resid, scale="normal") + 1e-7 

262 keep = np.abs(resid) <= self.config.numSigmaClip * apCorrErr 

263 

264 self.log.debug("Removing %d sources as outliers.", len(resid) - keep.sum()) 

265 

266 x = x[keep] 

267 y = y[keep] 

268 z = z[keep] 

269 ids = ids[keep] 

270 

271 while (len(x) - ctrl.computeSize()) < self.config.minDegreesOfFreedom: 

272 if ctrl.orderX > 0: 

273 ctrl.orderX -= 1 

274 else: 

275 allBad = True 

276 break 

277 if ctrl.orderY > 0: 

278 ctrl.orderY -= 1 

279 else: 

280 allBad = True 

281 break 

282 

283 if allBad: 

284 if name in self.config.allowFailure: 

285 self.log.warning("Unable to measure aperture correction for '%s': " 

286 "only %d sources remain, but require at least %d." % 

287 (name, keep.sum(), self.config.minDegreesOfFreedom + 1)) 

288 break 

289 else: 

290 msg = ("Unable to measure aperture correction for required algorithm " 

291 "'%s': only %d sources remain, but require at least %d." % 

292 (name, keep.sum(), self.config.minDegreesOfFreedom + 1)) 

293 self.log.warning(msg) 

294 raise MeasureApCorrError("Aperture correction failed on required algorithm.") 

295 

296 apCorrField = ChebyshevBoundedField.fit(bbox, x, y, z, ctrl) 

297 fitValues = apCorrField.evaluate(x, y) 

298 

299 if allBad: 

300 continue 

301 

302 self.log.info( 

303 "Aperture correction for %s from %d stars: MAD %f, RMS %f", 

304 name, 

305 median_abs_deviation(fitValues - z, scale="normal"), 

306 np.mean((fitValues - z)**2.)**0.5, 

307 len(x), 

308 ) 

309 

310 if display: 

311 plotApCorr(bbox, x, y, z, apCorrField, "%s, final" % (name,), doPause) 

312 

313 # Record which sources were used. 

314 used = np.zeros(len(catalog), dtype=bool) 

315 used[np.searchsorted(catalog['id'], ids)] = True 

316 catalog[fluxNames.usedName] = used 

317 

318 # Save the result in the output map 

319 # The error is constant spatially (we could imagine being 

320 # more clever, but we're not yet sure if it's worth the effort). 

321 # We save the errors as a 0th-order ChebyshevBoundedField 

322 apCorrMap[fluxNames.fluxName] = apCorrField 

323 apCorrMap[fluxNames.errName] = ChebyshevBoundedField( 

324 bbox, 

325 np.array([[apCorrErr]]), 

326 ) 

327 

328 return Struct( 

329 apCorrMap=apCorrMap, 

330 ) 

331 

332 

333def plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause): 

334 """Plot aperture correction fit residuals 

335 

336 There are two subplots: residuals against x and y. 

337 

338 Intended for debugging. 

339 

340 Parameters 

341 ---------- 

342 bbox : `lsst.geom.Box2I` 

343 Bounding box (for bounds) 

344 xx, yy : `numpy.ndarray`, (N) 

345 x and y coordinates 

346 zzMeasure : `float` 

347 Measured value of the aperture correction 

348 field : `lsst.afw.math.ChebyshevBoundedField` 

349 Fit aperture correction field 

350 title : 'str' 

351 Title for plot 

352 doPause : `bool` 

353 Pause to inspect the residuals plot? If 

354 False, there will be a 4 second delay to 

355 allow for inspection of the plot before 

356 closing it and moving on. 

357 """ 

358 import matplotlib.pyplot as plt 

359 

360 zzFit = field.evaluate(xx, yy) 

361 residuals = zzMeasure - zzFit 

362 

363 fig, axes = plt.subplots(2, 1) 

364 

365 axes[0].scatter(xx, residuals, s=3, marker='o', lw=0, alpha=0.7) 

366 axes[1].scatter(yy, residuals, s=3, marker='o', lw=0, alpha=0.7) 

367 for ax in axes: 

368 ax.set_ylabel("ApCorr Fit Residual") 

369 ax.set_ylim(0.9*residuals.min(), 1.1*residuals.max()) 

370 axes[0].set_xlabel("x") 

371 axes[0].set_xlim(bbox.getMinX(), bbox.getMaxX()) 

372 axes[1].set_xlabel("y") 

373 axes[1].set_xlim(bbox.getMinY(), bbox.getMaxY()) 

374 plt.suptitle(title) 

375 

376 if not doPause: 

377 try: 

378 plt.pause(4) 

379 plt.close() 

380 except Exception: 

381 print("%s: plt.pause() failed. Please close plots when done." % __name__) 

382 plt.show() 

383 else: 

384 print("%s: Please close plots when done." % __name__) 

385 plt.show()