lsst.meas.algorithms gf82c78fb6f+17a944dac7
Loading...
Searching...
No Matches
measureApCorr.py
Go to the documentation of this file.
2# LSST Data Management System
3#
4# Copyright 2008-2017 AURA/LSST.
5#
6# This product includes software developed by the
7# LSST Project (http://www.lsst.org/).
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the LSST License Statement and
20# the GNU General Public License along with this program. If not,
21# see <https://www.lsstcorp.org/LegalNotices/>.
22#
23
24__all__ = ("MeasureApCorrConfig", "MeasureApCorrTask")
25
26import numpy as np
27from scipy.stats import median_abs_deviation
28
29import lsst.pex.config
30from lsst.afw.image import ApCorrMap
31from lsst.afw.math import ChebyshevBoundedField, ChebyshevBoundedFieldConfig
32from lsst.pipe.base import Task, Struct
33from lsst.meas.base.apCorrRegistry import getApCorrNameSet
34
35from .sourceSelector import sourceSelectorRegistry
36
37
39 """A collection of flux-related names for a given flux measurement algorithm.
40
41 Parameters
42 ----------
43 name : `str`
44 Name of flux measurement algorithm, e.g. ``base_PsfFlux``.
45 schema : `lsst.afw.table.Schema`
46 Catalog schema containing the flux field. The ``{name}_instFlux``,
47 ``{name}_instFluxErr``, ``{name}_flag`` fields are checked for
48 existence, and the ``apcorr_{name}_used`` field is added.
49
50 Raises
51 ------
52 KeyError if any of instFlux, instFluxErr, or flag fields is missing.
53 """
54 def __init__(self, name, schema):
55 self.fluxName = name + "_instFlux"
56 if self.fluxName not in schema:
57 raise KeyError("Could not find " + self.fluxName)
58 self.errName = name + "_instFluxErr"
59 if self.errName not in schema:
60 raise KeyError("Could not find " + self.errName)
61 self.flagName = name + "_flag"
62 if self.flagName not in schema:
63 raise KeyError("Cound not find " + self.flagName)
64 self.usedName = "apcorr_" + name + "_used"
65 schema.addField(self.usedName, type="Flag",
66 doc="Set if source was used in measuring aperture correction.")
67
68
69class MeasureApCorrConfig(lsst.pex.config.Config):
70 """Configuration for MeasureApCorrTask.
71 """
72 refFluxName = lsst.pex.config.Field(
73 doc="Field name prefix for the flux other measurements should be aperture corrected to match",
74 dtype=str,
75 default="slot_CalibFlux",
76 )
77 sourceSelector = sourceSelectorRegistry.makeField(
78 doc="Selector that sets the stars that aperture corrections will be measured from.",
79 default="science",
80 )
81 minDegreesOfFreedom = lsst.pex.config.RangeField(
82 doc="Minimum number of degrees of freedom (# of valid data points - # of parameters);"
83 " if this is exceeded, the order of the fit is decreased (in both dimensions), and"
84 " if we can't decrease it enough, we'll raise ValueError.",
85 dtype=int,
86 default=1,
87 min=1,
88 )
89 fitConfig = lsst.pex.config.ConfigField(
90 doc="Configuration used in fitting the aperture correction fields.",
91 dtype=ChebyshevBoundedFieldConfig,
92 )
93 numIter = lsst.pex.config.Field(
94 doc="Number of iterations for robust MAD sigma clipping.",
95 dtype=int,
96 default=4,
97 )
98 numSigmaClip = lsst.pex.config.Field(
99 doc="Number of robust MAD sigma to do clipping.",
100 dtype=float,
101 default=4.0,
102 )
103 allowFailure = lsst.pex.config.ListField(
104 doc="Allow these measurement algorithms to fail without an exception.",
105 dtype=str,
106 default=[],
107 )
108
109 def setDefaults(self):
110 selector = self.sourceSelector["science"]
111
112 selector.doFluxLimit = False
113 selector.doFlags = True
114 selector.doUnresolved = True
115 selector.doSignalToNoise = True
116 selector.doIsolated = False
117 selector.flags.good = []
118 selector.flags.bad = [
119 "base_PixelFlags_flag_edge",
120 "base_PixelFlags_flag_interpolatedCenter",
121 "base_PixelFlags_flag_saturatedCenter",
122 "base_PixelFlags_flag_crCenter",
123 "base_PixelFlags_flag_bad",
124 "base_PixelFlags_flag_interpolated",
125 "base_PixelFlags_flag_saturated",
126 ]
127 selector.signalToNoise.minimum = 200.0
128 selector.signalToNoise.maximum = None
129 selector.signalToNoise.fluxField = "base_PsfFlux_instFlux"
130 selector.signalToNoise.errField = "base_PsfFlux_instFluxErr"
131
132 def validate(self):
133 lsst.pex.config.Config.validate(self)
134 if self.sourceSelector.target.usesMatches:
135 raise lsst.pex.config.FieldValidationError(
136 MeasureApCorrConfig.sourceSelector,
137 self,
138 "Star selectors that require matches are not permitted.")
139
140
142 """Task to measure aperture correction.
143 """
144 ConfigClass = MeasureApCorrConfig
145 _DefaultName = "measureApCorr"
146
147 def __init__(self, schema, **kwargs):
148 """Construct a MeasureApCorrTask
149
150 For every name in lsst.meas.base.getApCorrNameSet():
151 - If the corresponding flux fields exist in the schema:
152 - Add a new field apcorr_{name}_used
153 - Add an entry to the self.toCorrect dict
154 - Otherwise silently skip the name
155 """
156 Task.__init__(self, **kwargs)
157 self.refFluxNames = _FluxNames(self.config.refFluxName, schema)
158 self.toCorrect = {} # dict of flux field name prefix: FluxKeys instance
159 for name in sorted(getApCorrNameSet()):
160 try:
161 self.toCorrect[name] = _FluxNames(name, schema)
162 except KeyError:
163 # if a field in the registry is missing, just ignore it.
164 pass
165 self.makeSubtask("sourceSelector")
166
167 def run(self, exposure, catalog):
168 """Measure aperture correction
169
170 Parameters
171 ----------
172 exposure : `lsst.afw.image.Exposure`
173 Exposure aperture corrections are being measured on. The
174 bounding box is retrieved from it, and it is passed to the
175 sourceSelector. The output aperture correction map is *not*
176 added to the exposure; this is left to the caller.
178 SourceCatalog containing measurements to be used to
179 compute aperture corrections.
180
181 Returns
182 -------
183 Struct : `lsst.pipe.base.Struct`
184 Contains the following:
185
186 ``apCorrMap``
187 aperture correction map (`lsst.afw.image.ApCorrMap`)
188 that contains two entries for each flux field:
189 - flux field (e.g. base_PsfFlux_instFlux): 2d model
190 - flux sigma field (e.g. base_PsfFlux_instFluxErr): 2d model of error
191 """
192 bbox = exposure.getBBox()
193 import lsstDebug
194 display = lsstDebug.Info(__name__).display
195 doPause = lsstDebug.Info(__name__).doPause
196
197 self.log.info("Measuring aperture corrections for %d flux fields", len(self.toCorrect))
198
199 # First, create a subset of the catalog that contains only selected stars
200 # with non-flagged reference fluxes.
201 selected = self.sourceSelector.run(catalog, exposure=exposure)
202
203 use = (
204 ~selected.sourceCat[self.refFluxNames.flagName]
205 & (np.isfinite(selected.sourceCat[self.refFluxNames.fluxName]))
206 )
207 goodRefCat = selected.sourceCat[use].copy()
208
209 apCorrMap = ApCorrMap()
210
211 # Outer loop over the fields we want to correct
212 for name, fluxNames in self.toCorrect.items():
213 # Create a more restricted subset with only the objects where the to-be-correct flux
214 # is not flagged.
215 fluxes = goodRefCat[fluxNames.fluxName]
216 with np.errstate(invalid="ignore"): # suppress NaN warnings.
217 isGood = (
218 (~goodRefCat[fluxNames.flagName])
219 & (np.isfinite(fluxes))
220 & (fluxes > 0.0)
221 )
222
223 # The 1 is the minimum number of ctrl.computeSize() when the order
224 # drops to 0 in both x and y.
225 if (isGood.sum() - 1) < self.config.minDegreesOfFreedom:
226 if name in self.config.allowFailure:
227 self.log.warning("Unable to measure aperture correction for '%s': "
228 "only %d sources, but require at least %d." %
229 (name, isGood.sum(), self.config.minDegreesOfFreedom + 1))
230 continue
231 else:
232 raise RuntimeError("Unable to measure aperture correction for required algorithm '%s': "
233 "only %d sources, but require at least %d." %
234 (name, isGood.sum(), self.config.minDegreesOfFreedom + 1))
235
236 goodCat = goodRefCat[isGood].copy()
237
238 x = goodCat['slot_Centroid_x']
239 y = goodCat['slot_Centroid_y']
240 z = goodCat[self.refFluxNames.fluxName]/goodCat[fluxNames.fluxName]
241 ids = goodCat['id']
242
243 # We start with an initial fit that is the median offset; this
244 # works well in practice.
245 fitValues = np.median(z)
246
247 ctrl = self.config.fitConfig.makeControl()
248
249 allBad = False
250 for iteration in range(self.config.numIter):
251 resid = z - fitValues
252 # We add a small (epsilon) amount of floating-point slop because
253 # the median_abs_deviation may give a value that is just larger than 0
254 # even if given a completely flat residual field (as in tests).
255 apCorrErr = median_abs_deviation(resid, scale="normal") + 1e-7
256 keep = np.abs(resid) <= self.config.numSigmaClip * apCorrErr
257
258 self.log.debug("Removing %d sources as outliers.", len(resid) - keep.sum())
259
260 x = x[keep]
261 y = y[keep]
262 z = z[keep]
263 ids = ids[keep]
264
265 while (len(x) - ctrl.computeSize()) < self.config.minDegreesOfFreedom:
266 if ctrl.orderX > 0:
267 ctrl.orderX -= 1
268 else:
269 allBad = True
270 break
271 if ctrl.orderY > 0:
272 ctrl.orderY -= 1
273 else:
274 allBad = True
275 break
276
277 if allBad:
278 if name in self.config.allowFailure:
279 self.log.warning("Unable to measure aperture correction for '%s': "
280 "only %d sources remain, but require at least %d." %
281 (name, keep.sum(), self.config.minDegreesOfFreedom + 1))
282 break
283 else:
284 raise RuntimeError("Unable to measure aperture correction for required algorithm "
285 "'%s': only %d sources remain, but require at least %d." %
286 (name, keep.sum(), self.config.minDegreesOfFreedom + 1))
287
288 apCorrField = ChebyshevBoundedField.fit(bbox, x, y, z, ctrl)
289 fitValues = apCorrField.evaluate(x, y)
290
291 if allBad:
292 continue
293
294 self.log.info(
295 "Aperture correction for %s from %d stars: MAD %f, RMS %f",
296 name,
297 median_abs_deviation(fitValues - z, scale="normal"),
298 np.mean((fitValues - z)**2.)**0.5,
299 len(x),
300 )
301
302 if display:
303 plotApCorr(bbox, x, y, z, apCorrField, "%s, final" % (name,), doPause)
304
305 # Record which sources were used.
306 used = np.zeros(len(catalog), dtype=bool)
307 used[np.searchsorted(catalog['id'], ids)] = True
308 catalog[fluxNames.usedName] = used
309
310 # Save the result in the output map
311 # The error is constant spatially (we could imagine being
312 # more clever, but we're not yet sure if it's worth the effort).
313 # We save the errors as a 0th-order ChebyshevBoundedField
314 apCorrMap[fluxNames.fluxName] = apCorrField
315 apCorrMap[fluxNames.errName] = ChebyshevBoundedField(
316 bbox,
317 np.array([[apCorrErr]]),
318 )
319
320 return Struct(
321 apCorrMap=apCorrMap,
322 )
323
324
325def plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause):
326 """Plot aperture correction fit residuals
327
328 There are two subplots: residuals against x and y.
329
330 Intended for debugging.
331
332 Parameters
333 ----------
334 bbox : `lsst.geom.Box2I`
335 Bounding box (for bounds)
336 xx, yy : `numpy.ndarray`, (N)
337 x and y coordinates
338 zzMeasure : `float`
339 Measured value of the aperture correction
341 Fit aperture correction field
342 title : 'str'
343 Title for plot
344 doPause : `bool`
345 Pause to inspect the residuals plot? If
346 False, there will be a 4 second delay to
347 allow for inspection of the plot before
348 closing it and moving on.
349 """
350 import matplotlib.pyplot as plt
351
352 zzFit = field.evaluate(xx, yy)
353 residuals = zzMeasure - zzFit
354
355 fig, axes = plt.subplots(2, 1)
356
357 axes[0].scatter(xx, residuals, s=3, marker='o', lw=0, alpha=0.7)
358 axes[1].scatter(yy, residuals, s=3, marker='o', lw=0, alpha=0.7)
359 for ax in axes:
360 ax.set_ylabel("ApCorr Fit Residual")
361 ax.set_ylim(0.9*residuals.min(), 1.1*residuals.max())
362 axes[0].set_xlabel("x")
363 axes[0].set_xlim(bbox.getMinX(), bbox.getMaxX())
364 axes[1].set_xlabel("y")
365 axes[1].set_xlim(bbox.getMinY(), bbox.getMaxY())
366 plt.suptitle(title)
367
368 if not doPause:
369 try:
370 plt.pause(4)
371 plt.close()
372 except Exception:
373 print("%s: plt.pause() failed. Please close plots when done." % __name__)
374 plt.show()
375 else:
376 print("%s: Please close plots when done." % __name__)
377 plt.show()
def plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause)