Coverage for python/lsst/meas/algorithms/measureApCorr.py: 19%

122 statements  

« prev     ^ index     » next       coverage.py v6.4.2, created at 2022-07-20 03:06 -0700

1# 

2# LSST Data Management System 

3# 

4# Copyright 2008-2017 AURA/LSST. 

5# 

6# This product includes software developed by the 

7# LSST Project (http://www.lsst.org/). 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the LSST License Statement and 

20# the GNU General Public License along with this program. If not, 

21# see <https://www.lsstcorp.org/LegalNotices/>. 

22# 

23 

24__all__ = ("MeasureApCorrConfig", "MeasureApCorrTask") 

25 

26import numpy 

27 

28import lsst.pex.config 

29from lsst.afw.image import ApCorrMap 

30from lsst.afw.math import ChebyshevBoundedField, ChebyshevBoundedFieldConfig 

31from lsst.pipe.base import Task, Struct 

32from lsst.meas.base.apCorrRegistry import getApCorrNameSet 

33 

34from .sourceSelector import sourceSelectorRegistry 

35 

36 

37class FluxKeys: 

38 """A collection of keys for a given flux measurement algorithm 

39 """ 

40 __slots__ = ("flux", "err", "flag", "used") # prevent accidentally adding fields 

41 

42 def __init__(self, name, schema): 

43 """Construct a FluxKeys 

44 

45 Parameters 

46 ---------- 

47 name : `str` 

48 Name of flux measurement algorithm, e.g. "base_PsfFlux" 

49 schema : `lsst.afw.table.Schema` 

50 Catalog schema containing the flux field 

51 read: {name}_instFlux, {name}_instFluxErr, {name}_flag 

52 added: apcorr_{name}_used 

53 """ 

54 self.flux = schema.find(name + "_instFlux").key 

55 self.err = schema.find(name + "_instFluxErr").key 

56 self.flag = schema.find(name + "_flag").key 

57 self.used = schema.addField("apcorr_" + name + "_used", type="Flag", 

58 doc="set if source was used in measuring aperture correction") 

59 

60 

61class MeasureApCorrConfig(lsst.pex.config.Config): 

62 """Configuration for MeasureApCorrTask 

63 """ 

64 refFluxName = lsst.pex.config.Field( 

65 doc="Field name prefix for the flux other measurements should be aperture corrected to match", 

66 dtype=str, 

67 default="slot_CalibFlux", 

68 ) 

69 sourceSelector = sourceSelectorRegistry.makeField( 

70 doc="Selector that sets the stars that aperture corrections will be measured from.", 

71 default="flagged", 

72 ) 

73 minDegreesOfFreedom = lsst.pex.config.RangeField( 

74 doc="Minimum number of degrees of freedom (# of valid data points - # of parameters);" 

75 " if this is exceeded, the order of the fit is decreased (in both dimensions), and" 

76 " if we can't decrease it enough, we'll raise ValueError.", 

77 dtype=int, 

78 default=1, 

79 min=1, 

80 ) 

81 fitConfig = lsst.pex.config.ConfigField( 

82 doc="Configuration used in fitting the aperture correction fields", 

83 dtype=ChebyshevBoundedFieldConfig, 

84 ) 

85 numIter = lsst.pex.config.Field( 

86 doc="Number of iterations for sigma clipping", 

87 dtype=int, 

88 default=4, 

89 ) 

90 numSigmaClip = lsst.pex.config.Field( 

91 doc="Number of standard devisations to clip at", 

92 dtype=float, 

93 default=3.0, 

94 ) 

95 allowFailure = lsst.pex.config.ListField( 

96 doc="Allow these measurement algorithms to fail without an exception", 

97 dtype=str, 

98 default=[], 

99 ) 

100 

101 def validate(self): 

102 lsst.pex.config.Config.validate(self) 

103 if self.sourceSelector.target.usesMatches: 

104 raise lsst.pex.config.FieldValidationError( 

105 MeasureApCorrConfig.sourceSelector, 

106 self, 

107 "Star selectors that require matches are not permitted") 

108 

109 

110class MeasureApCorrTask(Task): 

111 r"""Task to measure aperture correction 

112 """ 

113 ConfigClass = MeasureApCorrConfig 

114 _DefaultName = "measureApCorr" 

115 

116 def __init__(self, schema, **kwds): 

117 """Construct a MeasureApCorrTask 

118 

119 For every name in lsst.meas.base.getApCorrNameSet(): 

120 - If the corresponding flux fields exist in the schema: 

121 - Add a new field apcorr_{name}_used 

122 - Add an entry to the self.toCorrect dict 

123 - Otherwise silently skip the name 

124 """ 

125 Task.__init__(self, **kwds) 

126 self.refFluxKeys = FluxKeys(self.config.refFluxName, schema) 

127 self.toCorrect = {} # dict of flux field name prefix: FluxKeys instance 

128 for name in sorted(getApCorrNameSet()): 

129 try: 

130 self.toCorrect[name] = FluxKeys(name, schema) 

131 except KeyError: 

132 # if a field in the registry is missing, just ignore it. 

133 pass 

134 self.makeSubtask("sourceSelector") 

135 

136 def run(self, exposure, catalog): 

137 """Measure aperture correction 

138 

139 Parameters 

140 ---------- 

141 exposure : `lsst.afw.image.Exposure` 

142 Exposure aperture corrections are being measured on. The 

143 bounding box is retrieved from it, and it is passed to the 

144 sourceSelector. The output aperture correction map is *not* 

145 added to the exposure; this is left to the caller. 

146 catalog : `lsst.afw.table.SourceCatalog` 

147 SourceCatalog containing measurements to be used to 

148 compute aperture corrections. 

149 

150 Returns 

151 ------- 

152 Struct : `lsst.pipe.base.Struct` 

153 Contains the following: 

154 

155 ``apCorrMap`` 

156 aperture correction map (`lsst.afw.image.ApCorrMap`) 

157 that contains two entries for each flux field: 

158 - flux field (e.g. base_PsfFlux_instFlux): 2d model 

159 - flux sigma field (e.g. base_PsfFlux_instFluxErr): 2d model of error 

160 """ 

161 bbox = exposure.getBBox() 

162 import lsstDebug 

163 display = lsstDebug.Info(__name__).display 

164 doPause = lsstDebug.Info(__name__).doPause 

165 

166 self.log.info("Measuring aperture corrections for %d flux fields", len(self.toCorrect)) 

167 # First, create a subset of the catalog that contains only selected stars 

168 # with non-flagged reference fluxes. 

169 subset1 = [record for record in self.sourceSelector.run(catalog, exposure=exposure).sourceCat 

170 if (not record.get(self.refFluxKeys.flag) 

171 and numpy.isfinite(record.get(self.refFluxKeys.flux)))] 

172 

173 apCorrMap = ApCorrMap() 

174 

175 # Outer loop over the fields we want to correct 

176 for name, keys in self.toCorrect.items(): 

177 fluxName = name + "_instFlux" 

178 fluxErrName = name + "_instFluxErr" 

179 

180 # Create a more restricted subset with only the objects where the to-be-correct flux 

181 # is not flagged. 

182 fluxes = numpy.fromiter((record.get(keys.flux) for record in subset1), float) 

183 with numpy.errstate(invalid="ignore"): # suppress NAN warnings 

184 isGood = numpy.logical_and.reduce([ 

185 numpy.fromiter((not record.get(keys.flag) for record in subset1), bool), 

186 numpy.isfinite(fluxes), 

187 fluxes > 0.0, 

188 ]) 

189 subset2 = [record for record, good in zip(subset1, isGood) if good] 

190 

191 # Check that we have enough data points that we have at least the minimum of degrees of 

192 # freedom specified in the config. 

193 if len(subset2) - 1 < self.config.minDegreesOfFreedom: 

194 if name in self.config.allowFailure: 

195 self.log.warning("Unable to measure aperture correction for '%s': " 

196 "only %d sources, but require at least %d.", 

197 name, len(subset2), self.config.minDegreesOfFreedom + 1) 

198 continue 

199 raise RuntimeError("Unable to measure aperture correction for required algorithm '%s': " 

200 "only %d sources, but require at least %d." % 

201 (name, len(subset2), self.config.minDegreesOfFreedom + 1)) 

202 

203 # If we don't have enough data points to constrain the fit, reduce the order until we do 

204 ctrl = self.config.fitConfig.makeControl() 

205 while len(subset2) - ctrl.computeSize() < self.config.minDegreesOfFreedom: 

206 if ctrl.orderX > 0: 

207 ctrl.orderX -= 1 

208 if ctrl.orderY > 0: 

209 ctrl.orderY -= 1 

210 

211 # Fill numpy arrays with positions and the ratio of the reference flux to the to-correct flux 

212 x = numpy.zeros(len(subset2), dtype=float) 

213 y = numpy.zeros(len(subset2), dtype=float) 

214 apCorrData = numpy.zeros(len(subset2), dtype=float) 

215 indices = numpy.arange(len(subset2), dtype=int) 

216 for n, record in enumerate(subset2): 

217 x[n] = record.getX() 

218 y[n] = record.getY() 

219 apCorrData[n] = record.get(self.refFluxKeys.flux)/record.get(keys.flux) 

220 

221 for _i in range(self.config.numIter): 

222 

223 # Do the fit, save it in the output map 

224 apCorrField = ChebyshevBoundedField.fit(bbox, x, y, apCorrData, ctrl) 

225 

226 if display: 

227 plotApCorr(bbox, x, y, apCorrData, apCorrField, "%s, iteration %d" % (name, _i), doPause) 

228 

229 # Compute errors empirically, using the RMS difference between the true reference flux and the 

230 # corrected to-be-corrected flux. 

231 apCorrDiffs = apCorrField.evaluate(x, y) 

232 apCorrDiffs -= apCorrData 

233 apCorrErr = numpy.mean(apCorrDiffs**2)**0.5 

234 

235 # Clip bad data points 

236 apCorrDiffLim = self.config.numSigmaClip * apCorrErr 

237 with numpy.errstate(invalid="ignore"): # suppress NAN warning 

238 keep = numpy.fabs(apCorrDiffs) <= apCorrDiffLim 

239 x = x[keep] 

240 y = y[keep] 

241 apCorrData = apCorrData[keep] 

242 indices = indices[keep] 

243 

244 # Final fit after clipping 

245 apCorrField = ChebyshevBoundedField.fit(bbox, x, y, apCorrData, ctrl) 

246 

247 self.log.info("Aperture correction for %s: RMS %f from %d", 

248 name, numpy.mean((apCorrField.evaluate(x, y) - apCorrData)**2)**0.5, len(indices)) 

249 

250 if display: 

251 plotApCorr(bbox, x, y, apCorrData, apCorrField, "%s, final" % (name,), doPause) 

252 

253 # Save the result in the output map 

254 # The error is constant spatially (we could imagine being 

255 # more clever, but we're not yet sure if it's worth the effort). 

256 # We save the errors as a 0th-order ChebyshevBoundedField 

257 apCorrMap[fluxName] = apCorrField 

258 apCorrErrCoefficients = numpy.array([[apCorrErr]], dtype=float) 

259 apCorrMap[fluxErrName] = ChebyshevBoundedField(bbox, apCorrErrCoefficients) 

260 

261 # Record which sources were used 

262 for i in indices: 

263 subset2[i].set(keys.used, True) 

264 

265 return Struct( 

266 apCorrMap=apCorrMap, 

267 ) 

268 

269 

270def plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause): 

271 """Plot aperture correction fit residuals 

272 

273 There are two subplots: residuals against x and y. 

274 

275 Intended for debugging. 

276 

277 Parameters 

278 ---------- 

279 bbox : `lsst.geom.Box2I` 

280 Bounding box (for bounds) 

281 xx, yy : `numpy.ndarray`, (N) 

282 x and y coordinates 

283 zzMeasure : `float` 

284 Measured value of the aperture correction 

285 field : `lsst.afw.math.ChebyshevBoundedField` 

286 Fit aperture correction field 

287 title : 'str' 

288 Title for plot 

289 doPause : `bool` 

290 Pause to inspect the residuals plot? If 

291 False, there will be a 4 second delay to 

292 allow for inspection of the plot before 

293 closing it and moving on. 

294 """ 

295 import matplotlib.pyplot as plt 

296 

297 zzFit = field.evaluate(xx, yy) 

298 residuals = zzMeasure - zzFit 

299 

300 fig, axes = plt.subplots(2, 1) 

301 

302 axes[0].scatter(xx, residuals, s=3, marker='o', lw=0, alpha=0.7) 

303 axes[1].scatter(yy, residuals, s=3, marker='o', lw=0, alpha=0.7) 

304 for ax in axes: 

305 ax.set_ylabel("ApCorr Fit Residual") 

306 ax.set_ylim(0.9*residuals.min(), 1.1*residuals.max()) 

307 axes[0].set_xlabel("x") 

308 axes[0].set_xlim(bbox.getMinX(), bbox.getMaxX()) 

309 axes[1].set_xlabel("y") 

310 axes[1].set_xlim(bbox.getMinY(), bbox.getMaxY()) 

311 plt.suptitle(title) 

312 

313 if not doPause: 

314 try: 

315 plt.pause(4) 

316 plt.close() 

317 except Exception: 

318 print("%s: plt.pause() failed. Please close plots when done." % __name__) 

319 plt.show() 

320 else: 

321 print("%s: Please close plots when done." % __name__) 

322 plt.show()