Coverage for python/lsst/meas/algorithms/measureApCorr.py: 15%

152 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-04 12:18 +0000

1# 

2# LSST Data Management System 

3# 

4# Copyright 2008-2017 AURA/LSST. 

5# 

6# This product includes software developed by the 

7# LSST Project (http://www.lsst.org/). 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <https://www.lsstcorp.org/LegalNotices/>. 

22# 

23 

24__all__ = ("MeasureApCorrConfig", "MeasureApCorrTask", "MeasureApCorrError") 

25 

26import numpy as np 

27from scipy.stats import median_abs_deviation 

28 

29import lsst.pex.config 

30from lsst.afw.image import ApCorrMap 

31from lsst.afw.math import ChebyshevBoundedField, ChebyshevBoundedFieldConfig 

32from lsst.pipe.base import Task, Struct 

33from lsst.meas.base.apCorrRegistry import getApCorrNameSet 

34 

35from .sourceSelector import sourceSelectorRegistry 

36 

37 

38class MeasureApCorrError(RuntimeError): 

39 pass 

40 

41 

42class _FluxNames: 

43 """A collection of flux-related names for a given flux measurement algorithm. 

44 

45 Parameters 

46 ---------- 

47 name : `str` 

48 Name of flux measurement algorithm, e.g. ``base_PsfFlux``. 

49 schema : `lsst.afw.table.Schema` 

50 Catalog schema containing the flux field. The ``{name}_instFlux``, 

51 ``{name}_instFluxErr``, ``{name}_flag`` fields are checked for 

52 existence, and the ``apcorr_{name}_used`` field is added. 

53 

54 Raises 

55 ------ 

56 KeyError if any of instFlux, instFluxErr, or flag fields is missing. 

57 """ 

58 def __init__(self, name, schema): 

59 self.fluxName = name + "_instFlux" 

60 if self.fluxName not in schema: 

61 raise KeyError("Could not find " + self.fluxName) 

62 self.errName = name + "_instFluxErr" 

63 if self.errName not in schema: 

64 raise KeyError("Could not find " + self.errName) 

65 self.flagName = name + "_flag" 

66 if self.flagName not in schema: 

67 raise KeyError("Cound not find " + self.flagName) 

68 self.usedName = "apcorr_" + name + "_used" 

69 schema.addField(self.usedName, type="Flag", 

70 doc="Set if source was used in measuring aperture correction.") 

71 

72 

73class MeasureApCorrConfig(lsst.pex.config.Config): 

74 """Configuration for MeasureApCorrTask. 

75 """ 

76 refFluxName = lsst.pex.config.Field( 

77 doc="Field name prefix for the flux other measurements should be aperture corrected to match", 

78 dtype=str, 

79 default="slot_CalibFlux", 

80 ) 

81 sourceSelector = sourceSelectorRegistry.makeField( 

82 doc="Selector that sets the stars that aperture corrections will be measured from.", 

83 default="science", 

84 ) 

85 minDegreesOfFreedom = lsst.pex.config.RangeField( 

86 doc="Minimum number of degrees of freedom (# of valid data points - # of parameters);" 

87 " if this is exceeded, the order of the fit is decreased (in both dimensions), and" 

88 " if we can't decrease it enough, we'll raise ValueError.", 

89 dtype=int, 

90 default=1, 

91 min=1, 

92 ) 

93 fitConfig = lsst.pex.config.ConfigField( 

94 doc="Configuration used in fitting the aperture correction fields.", 

95 dtype=ChebyshevBoundedFieldConfig, 

96 ) 

97 numIter = lsst.pex.config.Field( 

98 doc="Number of iterations for robust MAD sigma clipping.", 

99 dtype=int, 

100 default=4, 

101 ) 

102 numSigmaClip = lsst.pex.config.Field( 

103 doc="Number of robust MAD sigma to do clipping.", 

104 dtype=float, 

105 default=4.0, 

106 ) 

107 allowFailure = lsst.pex.config.ListField( 

108 doc="Allow these measurement algorithms to fail without an exception.", 

109 dtype=str, 

110 default=[], 

111 ) 

112 

113 def setDefaults(self): 

114 selector = self.sourceSelector["science"] 

115 

116 selector.doFlags = True 

117 selector.doUnresolved = True 

118 selector.doSignalToNoise = True 

119 selector.doIsolated = False 

120 selector.flags.good = [] 

121 selector.flags.bad = [ 

122 "base_PixelFlags_flag_edge", 

123 "base_PixelFlags_flag_interpolatedCenter", 

124 "base_PixelFlags_flag_saturatedCenter", 

125 "base_PixelFlags_flag_crCenter", 

126 "base_PixelFlags_flag_bad", 

127 "base_PixelFlags_flag_interpolated", 

128 "base_PixelFlags_flag_saturated", 

129 ] 

130 selector.signalToNoise.minimum = 200.0 

131 selector.signalToNoise.maximum = None 

132 selector.signalToNoise.fluxField = "base_PsfFlux_instFlux" 

133 selector.signalToNoise.errField = "base_PsfFlux_instFluxErr" 

134 

135 def validate(self): 

136 lsst.pex.config.Config.validate(self) 

137 if self.sourceSelector.target.usesMatches: 

138 raise lsst.pex.config.FieldValidationError( 

139 MeasureApCorrConfig.sourceSelector, 

140 self, 

141 "Star selectors that require matches are not permitted.") 

142 

143 

144class MeasureApCorrTask(Task): 

145 """Task to measure aperture correction. 

146 """ 

147 ConfigClass = MeasureApCorrConfig 

148 _DefaultName = "measureApCorr" 

149 

150 def __init__(self, schema, **kwargs): 

151 """Construct a MeasureApCorrTask 

152 

153 For every name in lsst.meas.base.getApCorrNameSet(): 

154 - If the corresponding flux fields exist in the schema: 

155 - Add a new field apcorr_{name}_used 

156 - Add an entry to the self.toCorrect dict 

157 - Otherwise silently skip the name 

158 """ 

159 Task.__init__(self, **kwargs) 

160 self.refFluxNames = _FluxNames(self.config.refFluxName, schema) 

161 self.toCorrect = {} # dict of flux field name prefix: FluxKeys instance 

162 for name in sorted(getApCorrNameSet()): 

163 try: 

164 self.toCorrect[name] = _FluxNames(name, schema) 

165 except KeyError: 

166 # if a field in the registry is missing, just ignore it. 

167 pass 

168 self.makeSubtask("sourceSelector") 

169 

170 def run(self, exposure, catalog): 

171 """Measure aperture correction 

172 

173 Parameters 

174 ---------- 

175 exposure : `lsst.afw.image.Exposure` 

176 Exposure aperture corrections are being measured on. The 

177 bounding box is retrieved from it, and it is passed to the 

178 sourceSelector. The output aperture correction map is *not* 

179 added to the exposure; this is left to the caller. 

180 catalog : `lsst.afw.table.SourceCatalog` 

181 SourceCatalog containing measurements to be used to 

182 compute aperture corrections. 

183 

184 Returns 

185 ------- 

186 Struct : `lsst.pipe.base.Struct` 

187 Contains the following: 

188 

189 ``apCorrMap`` 

190 aperture correction map (`lsst.afw.image.ApCorrMap`) 

191 that contains two entries for each flux field: 

192 - flux field (e.g. base_PsfFlux_instFlux): 2d model 

193 - flux sigma field (e.g. base_PsfFlux_instFluxErr): 2d model of error 

194 """ 

195 bbox = exposure.getBBox() 

196 import lsstDebug 

197 display = lsstDebug.Info(__name__).display 

198 doPause = lsstDebug.Info(__name__).doPause 

199 

200 self.log.info("Measuring aperture corrections for %d flux fields", len(self.toCorrect)) 

201 

202 # First, create a subset of the catalog that contains only selected stars 

203 # with non-flagged reference fluxes. 

204 selected = self.sourceSelector.run(catalog, exposure=exposure) 

205 

206 use = ( 

207 ~selected.sourceCat[self.refFluxNames.flagName] 

208 & (np.isfinite(selected.sourceCat[self.refFluxNames.fluxName])) 

209 ) 

210 goodRefCat = selected.sourceCat[use].copy() 

211 

212 apCorrMap = ApCorrMap() 

213 

214 # Outer loop over the fields we want to correct 

215 for name, fluxNames in self.toCorrect.items(): 

216 # Create a more restricted subset with only the objects where the to-be-correct flux 

217 # is not flagged. 

218 fluxes = goodRefCat[fluxNames.fluxName] 

219 with np.errstate(invalid="ignore"): # suppress NaN warnings. 

220 isGood = ( 

221 (~goodRefCat[fluxNames.flagName]) 

222 & (np.isfinite(fluxes)) 

223 & (fluxes > 0.0) 

224 ) 

225 

226 # The 1 is the minimum number of ctrl.computeSize() when the order 

227 # drops to 0 in both x and y. 

228 if (isGood.sum() - 1) < self.config.minDegreesOfFreedom: 

229 if name in self.config.allowFailure: 

230 self.log.warning("Unable to measure aperture correction for '%s': " 

231 "only %d sources, but require at least %d.", 

232 name, isGood.sum(), self.config.minDegreesOfFreedom + 1) 

233 continue 

234 else: 

235 msg = ("Unable to measure aperture correction for required algorithm '%s': " 

236 "only %d sources, but require at least %d." % 

237 (name, isGood.sum(), self.config.minDegreesOfFreedom + 1)) 

238 self.log.warning(msg) 

239 raise MeasureApCorrError("Aperture correction failed on required algorithm.") 

240 

241 goodCat = goodRefCat[isGood].copy() 

242 

243 x = goodCat['slot_Centroid_x'] 

244 y = goodCat['slot_Centroid_y'] 

245 z = goodCat[self.refFluxNames.fluxName]/goodCat[fluxNames.fluxName] 

246 ids = goodCat['id'] 

247 

248 # We start with an initial fit that is the median offset; this 

249 # works well in practice. 

250 fitValues = np.median(z) 

251 

252 ctrl = self.config.fitConfig.makeControl() 

253 

254 allBad = False 

255 for iteration in range(self.config.numIter): 

256 resid = z - fitValues 

257 # We add a small (epsilon) amount of floating-point slop because 

258 # the median_abs_deviation may give a value that is just larger than 0 

259 # even if given a completely flat residual field (as in tests). 

260 apCorrErr = median_abs_deviation(resid, scale="normal") + 1e-7 

261 keep = np.abs(resid) <= self.config.numSigmaClip * apCorrErr 

262 

263 self.log.debug("Removing %d sources as outliers.", len(resid) - keep.sum()) 

264 

265 x = x[keep] 

266 y = y[keep] 

267 z = z[keep] 

268 ids = ids[keep] 

269 

270 while (len(x) - ctrl.computeSize()) < self.config.minDegreesOfFreedom: 

271 if ctrl.orderX > 0: 

272 ctrl.orderX -= 1 

273 else: 

274 allBad = True 

275 break 

276 if ctrl.orderY > 0: 

277 ctrl.orderY -= 1 

278 else: 

279 allBad = True 

280 break 

281 

282 if allBad: 

283 if name in self.config.allowFailure: 

284 self.log.warning("Unable to measure aperture correction for '%s': " 

285 "only %d sources remain, but require at least %d." % 

286 (name, keep.sum(), self.config.minDegreesOfFreedom + 1)) 

287 break 

288 else: 

289 msg = ("Unable to measure aperture correction for required algorithm " 

290 "'%s': only %d sources remain, but require at least %d." % 

291 (name, keep.sum(), self.config.minDegreesOfFreedom + 1)) 

292 self.log.warning(msg) 

293 raise MeasureApCorrError("Aperture correction failed on required algorithm.") 

294 

295 apCorrField = ChebyshevBoundedField.fit(bbox, x, y, z, ctrl) 

296 fitValues = apCorrField.evaluate(x, y) 

297 

298 if allBad: 

299 continue 

300 

301 self.log.info( 

302 "Aperture correction for %s from %d stars: MAD %f, RMS %f", 

303 name, 

304 median_abs_deviation(fitValues - z, scale="normal"), 

305 np.mean((fitValues - z)**2.)**0.5, 

306 len(x), 

307 ) 

308 

309 if display: 

310 plotApCorr(bbox, x, y, z, apCorrField, "%s, final" % (name,), doPause) 

311 

312 # Record which sources were used. 

313 used = np.zeros(len(catalog), dtype=bool) 

314 used[np.searchsorted(catalog['id'], ids)] = True 

315 catalog[fluxNames.usedName] = used 

316 

317 # Save the result in the output map 

318 # The error is constant spatially (we could imagine being 

319 # more clever, but we're not yet sure if it's worth the effort). 

320 # We save the errors as a 0th-order ChebyshevBoundedField 

321 apCorrMap[fluxNames.fluxName] = apCorrField 

322 apCorrMap[fluxNames.errName] = ChebyshevBoundedField( 

323 bbox, 

324 np.array([[apCorrErr]]), 

325 ) 

326 

327 return Struct( 

328 apCorrMap=apCorrMap, 

329 ) 

330 

331 

332def plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause): 

333 """Plot aperture correction fit residuals 

334 

335 There are two subplots: residuals against x and y. 

336 

337 Intended for debugging. 

338 

339 Parameters 

340 ---------- 

341 bbox : `lsst.geom.Box2I` 

342 Bounding box (for bounds) 

343 xx, yy : `numpy.ndarray`, (N) 

344 x and y coordinates 

345 zzMeasure : `float` 

346 Measured value of the aperture correction 

347 field : `lsst.afw.math.ChebyshevBoundedField` 

348 Fit aperture correction field 

349 title : 'str' 

350 Title for plot 

351 doPause : `bool` 

352 Pause to inspect the residuals plot? If 

353 False, there will be a 4 second delay to 

354 allow for inspection of the plot before 

355 closing it and moving on. 

356 """ 

357 import matplotlib.pyplot as plt 

358 

359 zzFit = field.evaluate(xx, yy) 

360 residuals = zzMeasure - zzFit 

361 

362 fig, axes = plt.subplots(2, 1) 

363 

364 axes[0].scatter(xx, residuals, s=3, marker='o', lw=0, alpha=0.7) 

365 axes[1].scatter(yy, residuals, s=3, marker='o', lw=0, alpha=0.7) 

366 for ax in axes: 

367 ax.set_ylabel("ApCorr Fit Residual") 

368 ax.set_ylim(0.9*residuals.min(), 1.1*residuals.max()) 

369 axes[0].set_xlabel("x") 

370 axes[0].set_xlim(bbox.getMinX(), bbox.getMaxX()) 

371 axes[1].set_xlabel("y") 

372 axes[1].set_xlim(bbox.getMinY(), bbox.getMaxY()) 

373 plt.suptitle(title) 

374 

375 if not doPause: 

376 try: 

377 plt.pause(4) 

378 plt.close() 

379 except Exception: 

380 print("%s: plt.pause() failed. Please close plots when done." % __name__) 

381 plt.show() 

382 else: 

383 print("%s: Please close plots when done." % __name__) 

384 plt.show()