lsst.meas.algorithms gbe01a4569f+b2ef577126
measureApCorr.py
Go to the documentation of this file.
2# LSST Data Management System
3#
4# Copyright 2008-2017 AURA/LSST.
5#
6# This product includes software developed by the
7# LSST Project (http://www.lsst.org/).
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the LSST License Statement and
20# the GNU General Public License along with this program. If not,
21# see <https://www.lsstcorp.org/LegalNotices/>.
22#
23
24__all__ = ("MeasureApCorrConfig", "MeasureApCorrTask")
25
26import numpy
27
28import lsst.pex.config
29from lsst.afw.image import ApCorrMap
30from lsst.afw.math import ChebyshevBoundedField, ChebyshevBoundedFieldConfig
31from lsst.pipe.base import Task, Struct
32from lsst.meas.base.apCorrRegistry import getApCorrNameSet
33
34from .sourceSelector import sourceSelectorRegistry
35
36
38 """A collection of keys for a given flux measurement algorithm
39 """
40 __slots__ = ("flux", "err", "flag", "used") # prevent accidentally adding fields
41
42 def __init__(self, name, schema):
43 """Construct a FluxKeys
44
45 Parameters
46 ----------
47 name : `str`
48 Name of flux measurement algorithm, e.g. "base_PsfFlux"
49 schema : `lsst.afw.table.Schema`
50 Catalog schema containing the flux field
51 read: {name}_instFlux, {name}_instFluxErr, {name}_flag
52 added: apcorr_{name}_used
53 """
54 self.fluxflux = schema.find(name + "_instFlux").key
55 self.errerr = schema.find(name + "_instFluxErr").key
56 self.flagflag = schema.find(name + "_flag").key
57 self.usedused = schema.addField("apcorr_" + name + "_used", type="Flag",
58 doc="set if source was used in measuring aperture correction")
59
60
61class MeasureApCorrConfig(lsst.pex.config.Config):
62 """Configuration for MeasureApCorrTask
63 """
64 refFluxName = lsst.pex.config.Field(
65 doc="Field name prefix for the flux other measurements should be aperture corrected to match",
66 dtype=str,
67 default="slot_CalibFlux",
68 )
69 sourceSelector = sourceSelectorRegistry.makeField(
70 doc="Selector that sets the stars that aperture corrections will be measured from.",
71 default="flagged",
72 )
73 minDegreesOfFreedom = lsst.pex.config.RangeField(
74 doc="Minimum number of degrees of freedom (# of valid data points - # of parameters);"
75 " if this is exceeded, the order of the fit is decreased (in both dimensions), and"
76 " if we can't decrease it enough, we'll raise ValueError.",
77 dtype=int,
78 default=1,
79 min=1,
80 )
81 fitConfig = lsst.pex.config.ConfigField(
82 doc="Configuration used in fitting the aperture correction fields",
83 dtype=ChebyshevBoundedFieldConfig,
84 )
85 numIter = lsst.pex.config.Field(
86 doc="Number of iterations for sigma clipping",
87 dtype=int,
88 default=4,
89 )
90 numSigmaClip = lsst.pex.config.Field(
91 doc="Number of standard devisations to clip at",
92 dtype=float,
93 default=3.0,
94 )
95 allowFailure = lsst.pex.config.ListField(
96 doc="Allow these measurement algorithms to fail without an exception",
97 dtype=str,
98 default=[],
99 )
100
101 def validate(self):
102 lsst.pex.config.Config.validate(self)
103 if self.sourceSelectorsourceSelector.target.usesMatches:
104 raise lsst.pex.config.FieldValidationError(
105 MeasureApCorrConfig.sourceSelector,
106 self,
107 "Star selectors that require matches are not permitted")
108
109
111 r"""Task to measure aperture correction
112 """
113 ConfigClass = MeasureApCorrConfig
114 _DefaultName = "measureApCorr"
115
116 def __init__(self, schema, **kwds):
117 """Construct a MeasureApCorrTask
118
119 For every name in lsst.meas.base.getApCorrNameSet():
120 - If the corresponding flux fields exist in the schema:
121 - Add a new field apcorr_{name}_used
122 - Add an entry to the self.toCorrecttoCorrect dict
123 - Otherwise silently skip the name
124 """
125 Task.__init__(self, **kwds)
126 self.refFluxKeysrefFluxKeys = FluxKeys(self.config.refFluxName, schema)
127 self.toCorrecttoCorrect = {} # dict of flux field name prefix: FluxKeys instance
128 for name in sorted(getApCorrNameSet()):
129 try:
130 self.toCorrecttoCorrect[name] = FluxKeys(name, schema)
131 except KeyError:
132 # if a field in the registry is missing, just ignore it.
133 pass
134 self.makeSubtask("sourceSelector")
135
136 def run(self, exposure, catalog):
137 """Measure aperture correction
138
139 Parameters
140 ----------
141 exposure : `lsst.afw.image.Exposure`
142 Exposure aperture corrections are being measured on. The
143 bounding box is retrieved from it, and it is passed to the
144 sourceSelector. The output aperture correction map is *not*
145 added to the exposure; this is left to the caller.
147 SourceCatalog containing measurements to be used to
148 compute aperture corrections.
149
150 Returns
151 -------
152 Struct : `lsst.pipe.base.Struct`
153 Contains the following:
154
155 ``apCorrMap``
156 aperture correction map (`lsst.afw.image.ApCorrMap`)
157 that contains two entries for each flux field:
158 - flux field (e.g. base_PsfFlux_instFlux): 2d model
159 - flux sigma field (e.g. base_PsfFlux_instFluxErr): 2d model of error
160 """
161 bbox = exposure.getBBox()
162 import lsstDebug
163 display = lsstDebug.Info(__name__).display
164 doPause = lsstDebug.Info(__name__).doPause
165
166 self.log.info("Measuring aperture corrections for %d flux fields", len(self.toCorrecttoCorrect))
167 # First, create a subset of the catalog that contains only selected stars
168 # with non-flagged reference fluxes.
169 subset1 = [record for record in self.sourceSelector.run(catalog, exposure=exposure).sourceCat
170 if (not record.get(self.refFluxKeysrefFluxKeys.flag)
171 and numpy.isfinite(record.get(self.refFluxKeysrefFluxKeys.flux)))]
172
173 apCorrMap = ApCorrMap()
174
175 # Outer loop over the fields we want to correct
176 for name, keys in self.toCorrecttoCorrect.items():
177 fluxName = name + "_instFlux"
178 fluxErrName = name + "_instFluxErr"
179
180 # Create a more restricted subset with only the objects where the to-be-correct flux
181 # is not flagged.
182 fluxes = numpy.fromiter((record.get(keys.flux) for record in subset1), float)
183 with numpy.errstate(invalid="ignore"): # suppress NAN warnings
184 isGood = numpy.logical_and.reduce([
185 numpy.fromiter((not record.get(keys.flag) for record in subset1), bool),
186 numpy.isfinite(fluxes),
187 fluxes > 0.0,
188 ])
189 subset2 = [record for record, good in zip(subset1, isGood) if good]
190
191 # Check that we have enough data points that we have at least the minimum of degrees of
192 # freedom specified in the config.
193 if len(subset2) - 1 < self.config.minDegreesOfFreedom:
194 if name in self.config.allowFailure:
195 self.log.warning("Unable to measure aperture correction for '%s': "
196 "only %d sources, but require at least %d.",
197 name, len(subset2), self.config.minDegreesOfFreedom + 1)
198 continue
199 raise RuntimeError("Unable to measure aperture correction for required algorithm '%s': "
200 "only %d sources, but require at least %d." %
201 (name, len(subset2), self.config.minDegreesOfFreedom + 1))
202
203 # If we don't have enough data points to constrain the fit, reduce the order until we do
204 ctrl = self.config.fitConfig.makeControl()
205 while len(subset2) - ctrl.computeSize() < self.config.minDegreesOfFreedom:
206 if ctrl.orderX > 0:
207 ctrl.orderX -= 1
208 if ctrl.orderY > 0:
209 ctrl.orderY -= 1
210
211 # Fill numpy arrays with positions and the ratio of the reference flux to the to-correct flux
212 x = numpy.zeros(len(subset2), dtype=float)
213 y = numpy.zeros(len(subset2), dtype=float)
214 apCorrData = numpy.zeros(len(subset2), dtype=float)
215 indices = numpy.arange(len(subset2), dtype=int)
216 for n, record in enumerate(subset2):
217 x[n] = record.getX()
218 y[n] = record.getY()
219 apCorrData[n] = record.get(self.refFluxKeysrefFluxKeys.flux)/record.get(keys.flux)
220
221 for _i in range(self.config.numIter):
222
223 # Do the fit, save it in the output map
224 apCorrField = ChebyshevBoundedField.fit(bbox, x, y, apCorrData, ctrl)
225
226 if display:
227 plotApCorr(bbox, x, y, apCorrData, apCorrField, "%s, iteration %d" % (name, _i), doPause)
228
229 # Compute errors empirically, using the RMS difference between the true reference flux and the
230 # corrected to-be-corrected flux.
231 apCorrDiffs = apCorrField.evaluate(x, y)
232 apCorrDiffs -= apCorrData
233 apCorrErr = numpy.mean(apCorrDiffs**2)**0.5
234
235 # Clip bad data points
236 apCorrDiffLim = self.config.numSigmaClip * apCorrErr
237 with numpy.errstate(invalid="ignore"): # suppress NAN warning
238 keep = numpy.fabs(apCorrDiffs) <= apCorrDiffLim
239 x = x[keep]
240 y = y[keep]
241 apCorrData = apCorrData[keep]
242 indices = indices[keep]
243
244 # Final fit after clipping
245 apCorrField = ChebyshevBoundedField.fit(bbox, x, y, apCorrData, ctrl)
246
247 self.log.info("Aperture correction for %s: RMS %f from %d",
248 name, numpy.mean((apCorrField.evaluate(x, y) - apCorrData)**2)**0.5, len(indices))
249
250 if display:
251 plotApCorr(bbox, x, y, apCorrData, apCorrField, "%s, final" % (name,), doPause)
252
253 # Save the result in the output map
254 # The error is constant spatially (we could imagine being
255 # more clever, but we're not yet sure if it's worth the effort).
256 # We save the errors as a 0th-order ChebyshevBoundedField
257 apCorrMap[fluxName] = apCorrField
258 apCorrErrCoefficients = numpy.array([[apCorrErr]], dtype=float)
259 apCorrMap[fluxErrName] = ChebyshevBoundedField(bbox, apCorrErrCoefficients)
260
261 # Record which sources were used
262 for i in indices:
263 subset2[i].set(keys.used, True)
264
265 return Struct(
266 apCorrMap=apCorrMap,
267 )
268
269
270def plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause):
271 """Plot aperture correction fit residuals
272
273 There are two subplots: residuals against x and y.
274
275 Intended for debugging.
276
277 Parameters
278 ----------
279 bbox : `lsst.geom.Box2I`
280 Bounding box (for bounds)
281 xx, yy : `numpy.ndarray`, (N)
282 x and y coordinates
283 zzMeasure : `float`
284 Measured value of the aperture correction
286 Fit aperture correction field
287 title : 'str'
288 Title for plot
289 doPause : `bool`
290 Pause to inspect the residuals plot? If
291 False, there will be a 4 second delay to
292 allow for inspection of the plot before
293 closing it and moving on.
294 """
295 import matplotlib.pyplot as plt
296
297 zzFit = field.evaluate(xx, yy)
298 residuals = zzMeasure - zzFit
299
300 fig, axes = plt.subplots(2, 1)
301
302 axes[0].scatter(xx, residuals, s=3, marker='o', lw=0, alpha=0.7)
303 axes[1].scatter(yy, residuals, s=3, marker='o', lw=0, alpha=0.7)
304 for ax in axes:
305 ax.set_ylabel("ApCorr Fit Residual")
306 ax.set_ylim(0.9*residuals.min(), 1.1*residuals.max())
307 axes[0].set_xlabel("x")
308 axes[0].set_xlim(bbox.getMinX(), bbox.getMaxX())
309 axes[1].set_xlabel("y")
310 axes[1].set_xlim(bbox.getMinY(), bbox.getMaxY())
311 plt.suptitle(title)
312
313 if not doPause:
314 try:
315 plt.pause(4)
316 plt.close()
317 except Exception:
318 print("%s: plt.pause() failed. Please close plots when done." % __name__)
319 plt.show()
320 else:
321 print("%s: Please close plots when done." % __name__)
322 plt.show()
def plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause)