Coverage for python/lsst/meas/algorithms/measureApCorr.py: 19%
122 statements
« prev ^ index » next coverage.py v6.4.2, created at 2022-08-01 01:38 -0700
« prev ^ index » next coverage.py v6.4.2, created at 2022-08-01 01:38 -0700
1#
2# LSST Data Management System
3#
4# Copyright 2008-2017 AURA/LSST.
5#
6# This product includes software developed by the
7# LSST Project (http://www.lsst.org/).
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the LSST License Statement and
20# the GNU General Public License along with this program. If not,
21# see <https://www.lsstcorp.org/LegalNotices/>.
22#
24__all__ = ("MeasureApCorrConfig", "MeasureApCorrTask")
26import numpy
28import lsst.pex.config
29from lsst.afw.image import ApCorrMap
30from lsst.afw.math import ChebyshevBoundedField, ChebyshevBoundedFieldConfig
31from lsst.pipe.base import Task, Struct
32from lsst.meas.base.apCorrRegistry import getApCorrNameSet
34from .sourceSelector import sourceSelectorRegistry
37class FluxKeys:
38 """A collection of keys for a given flux measurement algorithm
39 """
40 __slots__ = ("flux", "err", "flag", "used") # prevent accidentally adding fields
42 def __init__(self, name, schema):
43 """Construct a FluxKeys
45 Parameters
46 ----------
47 name : `str`
48 Name of flux measurement algorithm, e.g. "base_PsfFlux"
49 schema : `lsst.afw.table.Schema`
50 Catalog schema containing the flux field
51 read: {name}_instFlux, {name}_instFluxErr, {name}_flag
52 added: apcorr_{name}_used
53 """
54 self.flux = schema.find(name + "_instFlux").key
55 self.err = schema.find(name + "_instFluxErr").key
56 self.flag = schema.find(name + "_flag").key
57 self.used = schema.addField("apcorr_" + name + "_used", type="Flag",
58 doc="set if source was used in measuring aperture correction")
61class MeasureApCorrConfig(lsst.pex.config.Config):
62 """Configuration for MeasureApCorrTask
63 """
64 refFluxName = lsst.pex.config.Field(
65 doc="Field name prefix for the flux other measurements should be aperture corrected to match",
66 dtype=str,
67 default="slot_CalibFlux",
68 )
69 sourceSelector = sourceSelectorRegistry.makeField(
70 doc="Selector that sets the stars that aperture corrections will be measured from.",
71 default="flagged",
72 )
73 minDegreesOfFreedom = lsst.pex.config.RangeField(
74 doc="Minimum number of degrees of freedom (# of valid data points - # of parameters);"
75 " if this is exceeded, the order of the fit is decreased (in both dimensions), and"
76 " if we can't decrease it enough, we'll raise ValueError.",
77 dtype=int,
78 default=1,
79 min=1,
80 )
81 fitConfig = lsst.pex.config.ConfigField(
82 doc="Configuration used in fitting the aperture correction fields",
83 dtype=ChebyshevBoundedFieldConfig,
84 )
85 numIter = lsst.pex.config.Field(
86 doc="Number of iterations for sigma clipping",
87 dtype=int,
88 default=4,
89 )
90 numSigmaClip = lsst.pex.config.Field(
91 doc="Number of standard devisations to clip at",
92 dtype=float,
93 default=3.0,
94 )
95 allowFailure = lsst.pex.config.ListField(
96 doc="Allow these measurement algorithms to fail without an exception",
97 dtype=str,
98 default=[],
99 )
101 def validate(self):
102 lsst.pex.config.Config.validate(self)
103 if self.sourceSelector.target.usesMatches:
104 raise lsst.pex.config.FieldValidationError(
105 MeasureApCorrConfig.sourceSelector,
106 self,
107 "Star selectors that require matches are not permitted")
110class MeasureApCorrTask(Task):
111 r"""Task to measure aperture correction
112 """
113 ConfigClass = MeasureApCorrConfig
114 _DefaultName = "measureApCorr"
116 def __init__(self, schema, **kwds):
117 """Construct a MeasureApCorrTask
119 For every name in lsst.meas.base.getApCorrNameSet():
120 - If the corresponding flux fields exist in the schema:
121 - Add a new field apcorr_{name}_used
122 - Add an entry to the self.toCorrect dict
123 - Otherwise silently skip the name
124 """
125 Task.__init__(self, **kwds)
126 self.refFluxKeys = FluxKeys(self.config.refFluxName, schema)
127 self.toCorrect = {} # dict of flux field name prefix: FluxKeys instance
128 for name in sorted(getApCorrNameSet()):
129 try:
130 self.toCorrect[name] = FluxKeys(name, schema)
131 except KeyError:
132 # if a field in the registry is missing, just ignore it.
133 pass
134 self.makeSubtask("sourceSelector")
136 def run(self, exposure, catalog):
137 """Measure aperture correction
139 Parameters
140 ----------
141 exposure : `lsst.afw.image.Exposure`
142 Exposure aperture corrections are being measured on. The
143 bounding box is retrieved from it, and it is passed to the
144 sourceSelector. The output aperture correction map is *not*
145 added to the exposure; this is left to the caller.
146 catalog : `lsst.afw.table.SourceCatalog`
147 SourceCatalog containing measurements to be used to
148 compute aperture corrections.
150 Returns
151 -------
152 Struct : `lsst.pipe.base.Struct`
153 Contains the following:
155 ``apCorrMap``
156 aperture correction map (`lsst.afw.image.ApCorrMap`)
157 that contains two entries for each flux field:
158 - flux field (e.g. base_PsfFlux_instFlux): 2d model
159 - flux sigma field (e.g. base_PsfFlux_instFluxErr): 2d model of error
160 """
161 bbox = exposure.getBBox()
162 import lsstDebug
163 display = lsstDebug.Info(__name__).display
164 doPause = lsstDebug.Info(__name__).doPause
166 self.log.info("Measuring aperture corrections for %d flux fields", len(self.toCorrect))
167 # First, create a subset of the catalog that contains only selected stars
168 # with non-flagged reference fluxes.
169 subset1 = [record for record in self.sourceSelector.run(catalog, exposure=exposure).sourceCat
170 if (not record.get(self.refFluxKeys.flag)
171 and numpy.isfinite(record.get(self.refFluxKeys.flux)))]
173 apCorrMap = ApCorrMap()
175 # Outer loop over the fields we want to correct
176 for name, keys in self.toCorrect.items():
177 fluxName = name + "_instFlux"
178 fluxErrName = name + "_instFluxErr"
180 # Create a more restricted subset with only the objects where the to-be-correct flux
181 # is not flagged.
182 fluxes = numpy.fromiter((record.get(keys.flux) for record in subset1), float)
183 with numpy.errstate(invalid="ignore"): # suppress NAN warnings
184 isGood = numpy.logical_and.reduce([
185 numpy.fromiter((not record.get(keys.flag) for record in subset1), bool),
186 numpy.isfinite(fluxes),
187 fluxes > 0.0,
188 ])
189 subset2 = [record for record, good in zip(subset1, isGood) if good]
191 # Check that we have enough data points that we have at least the minimum of degrees of
192 # freedom specified in the config.
193 if len(subset2) - 1 < self.config.minDegreesOfFreedom:
194 if name in self.config.allowFailure:
195 self.log.warning("Unable to measure aperture correction for '%s': "
196 "only %d sources, but require at least %d.",
197 name, len(subset2), self.config.minDegreesOfFreedom + 1)
198 continue
199 raise RuntimeError("Unable to measure aperture correction for required algorithm '%s': "
200 "only %d sources, but require at least %d." %
201 (name, len(subset2), self.config.minDegreesOfFreedom + 1))
203 # If we don't have enough data points to constrain the fit, reduce the order until we do
204 ctrl = self.config.fitConfig.makeControl()
205 while len(subset2) - ctrl.computeSize() < self.config.minDegreesOfFreedom:
206 if ctrl.orderX > 0:
207 ctrl.orderX -= 1
208 if ctrl.orderY > 0:
209 ctrl.orderY -= 1
211 # Fill numpy arrays with positions and the ratio of the reference flux to the to-correct flux
212 x = numpy.zeros(len(subset2), dtype=float)
213 y = numpy.zeros(len(subset2), dtype=float)
214 apCorrData = numpy.zeros(len(subset2), dtype=float)
215 indices = numpy.arange(len(subset2), dtype=int)
216 for n, record in enumerate(subset2):
217 x[n] = record.getX()
218 y[n] = record.getY()
219 apCorrData[n] = record.get(self.refFluxKeys.flux)/record.get(keys.flux)
221 for _i in range(self.config.numIter):
223 # Do the fit, save it in the output map
224 apCorrField = ChebyshevBoundedField.fit(bbox, x, y, apCorrData, ctrl)
226 if display:
227 plotApCorr(bbox, x, y, apCorrData, apCorrField, "%s, iteration %d" % (name, _i), doPause)
229 # Compute errors empirically, using the RMS difference between the true reference flux and the
230 # corrected to-be-corrected flux.
231 apCorrDiffs = apCorrField.evaluate(x, y)
232 apCorrDiffs -= apCorrData
233 apCorrErr = numpy.mean(apCorrDiffs**2)**0.5
235 # Clip bad data points
236 apCorrDiffLim = self.config.numSigmaClip * apCorrErr
237 with numpy.errstate(invalid="ignore"): # suppress NAN warning
238 keep = numpy.fabs(apCorrDiffs) <= apCorrDiffLim
239 x = x[keep]
240 y = y[keep]
241 apCorrData = apCorrData[keep]
242 indices = indices[keep]
244 # Final fit after clipping
245 apCorrField = ChebyshevBoundedField.fit(bbox, x, y, apCorrData, ctrl)
247 self.log.info("Aperture correction for %s: RMS %f from %d",
248 name, numpy.mean((apCorrField.evaluate(x, y) - apCorrData)**2)**0.5, len(indices))
250 if display:
251 plotApCorr(bbox, x, y, apCorrData, apCorrField, "%s, final" % (name,), doPause)
253 # Save the result in the output map
254 # The error is constant spatially (we could imagine being
255 # more clever, but we're not yet sure if it's worth the effort).
256 # We save the errors as a 0th-order ChebyshevBoundedField
257 apCorrMap[fluxName] = apCorrField
258 apCorrErrCoefficients = numpy.array([[apCorrErr]], dtype=float)
259 apCorrMap[fluxErrName] = ChebyshevBoundedField(bbox, apCorrErrCoefficients)
261 # Record which sources were used
262 for i in indices:
263 subset2[i].set(keys.used, True)
265 return Struct(
266 apCorrMap=apCorrMap,
267 )
270def plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause):
271 """Plot aperture correction fit residuals
273 There are two subplots: residuals against x and y.
275 Intended for debugging.
277 Parameters
278 ----------
279 bbox : `lsst.geom.Box2I`
280 Bounding box (for bounds)
281 xx, yy : `numpy.ndarray`, (N)
282 x and y coordinates
283 zzMeasure : `float`
284 Measured value of the aperture correction
285 field : `lsst.afw.math.ChebyshevBoundedField`
286 Fit aperture correction field
287 title : 'str'
288 Title for plot
289 doPause : `bool`
290 Pause to inspect the residuals plot? If
291 False, there will be a 4 second delay to
292 allow for inspection of the plot before
293 closing it and moving on.
294 """
295 import matplotlib.pyplot as plt
297 zzFit = field.evaluate(xx, yy)
298 residuals = zzMeasure - zzFit
300 fig, axes = plt.subplots(2, 1)
302 axes[0].scatter(xx, residuals, s=3, marker='o', lw=0, alpha=0.7)
303 axes[1].scatter(yy, residuals, s=3, marker='o', lw=0, alpha=0.7)
304 for ax in axes:
305 ax.set_ylabel("ApCorr Fit Residual")
306 ax.set_ylim(0.9*residuals.min(), 1.1*residuals.max())
307 axes[0].set_xlabel("x")
308 axes[0].set_xlim(bbox.getMinX(), bbox.getMaxX())
309 axes[1].set_xlabel("y")
310 axes[1].set_xlim(bbox.getMinY(), bbox.getMaxY())
311 plt.suptitle(title)
313 if not doPause:
314 try:
315 plt.pause(4)
316 plt.close()
317 except Exception:
318 print("%s: plt.pause() failed. Please close plots when done." % __name__)
319 plt.show()
320 else:
321 print("%s: Please close plots when done." % __name__)
322 plt.show()