lsst.meas.algorithms gee31e0d7c8+667bae79af
Loading...
Searching...
No Matches
measureApCorr.py
Go to the documentation of this file.
2# LSST Data Management System
3#
4# Copyright 2008-2017 AURA/LSST.
5#
6# This product includes software developed by the
7# LSST Project (http://www.lsst.org/).
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the LSST License Statement and
20# the GNU General Public License along with this program. If not,
21# see <https://www.lsstcorp.org/LegalNotices/>.
22#
23
24__all__ = ("MeasureApCorrConfig", "MeasureApCorrTask", "MeasureApCorrError")
25
26import numpy as np
27from scipy.stats import median_abs_deviation
28
29import lsst.pex.config
30from lsst.afw.image import ApCorrMap
31from lsst.afw.math import ChebyshevBoundedField, ChebyshevBoundedFieldConfig
32from lsst.pipe.base import Task, Struct
33from lsst.meas.base.apCorrRegistry import getApCorrNameSet
34
35from .sourceSelector import sourceSelectorRegistry
36
37
38class MeasureApCorrError(RuntimeError):
39 pass
40
41
43 """A collection of flux-related names for a given flux measurement algorithm.
44
45 Parameters
46 ----------
47 name : `str`
48 Name of flux measurement algorithm, e.g. ``base_PsfFlux``.
49 schema : `lsst.afw.table.Schema`
50 Catalog schema containing the flux field. The ``{name}_instFlux``,
51 ``{name}_instFluxErr``, ``{name}_flag`` fields are checked for
52 existence, and the ``apcorr_{name}_used`` field is added.
53
54 Raises
55 ------
56 KeyError if any of instFlux, instFluxErr, or flag fields is missing.
57 """
58 def __init__(self, name, schema):
59 self.fluxName = name + "_instFlux"
60 if self.fluxName not in schema:
61 raise KeyError("Could not find " + self.fluxName)
62 self.errName = name + "_instFluxErr"
63 if self.errName not in schema:
64 raise KeyError("Could not find " + self.errName)
65 self.flagName = name + "_flag"
66 if self.flagName not in schema:
67 raise KeyError("Cound not find " + self.flagName)
68 self.usedName = "apcorr_" + name + "_used"
69 schema.addField(self.usedName, type="Flag",
70 doc="Set if source was used in measuring aperture correction.")
71
72
73class MeasureApCorrConfig(lsst.pex.config.Config):
74 """Configuration for MeasureApCorrTask.
75 """
76 refFluxName = lsst.pex.config.Field(
77 doc="Field name prefix for the flux other measurements should be aperture corrected to match",
78 dtype=str,
79 default="slot_CalibFlux",
80 )
81 sourceSelector = sourceSelectorRegistry.makeField(
82 doc="Selector that sets the stars that aperture corrections will be measured from.",
83 default="science",
84 )
85 minDegreesOfFreedom = lsst.pex.config.RangeField(
86 doc="Minimum number of degrees of freedom (# of valid data points - # of parameters);"
87 " if this is exceeded, the order of the fit is decreased (in both dimensions), and"
88 " if we can't decrease it enough, we'll raise ValueError.",
89 dtype=int,
90 default=1,
91 min=1,
92 )
93 fitConfig = lsst.pex.config.ConfigField(
94 doc="Configuration used in fitting the aperture correction fields.",
95 dtype=ChebyshevBoundedFieldConfig,
96 )
97 numIter = lsst.pex.config.Field(
98 doc="Number of iterations for robust MAD sigma clipping.",
99 dtype=int,
100 default=4,
101 )
102 numSigmaClip = lsst.pex.config.Field(
103 doc="Number of robust MAD sigma to do clipping.",
104 dtype=float,
105 default=4.0,
106 )
107 allowFailure = lsst.pex.config.ListField(
108 doc="Allow these measurement algorithms to fail without an exception.",
109 dtype=str,
110 default=[],
111 )
112
113 def setDefaults(self):
114 selector = self.sourceSelector["science"]
115
116 selector.doFluxLimit = False
117 selector.doFlags = True
118 selector.doUnresolved = True
119 selector.doSignalToNoise = True
120 selector.doIsolated = False
121 selector.flags.good = []
122 selector.flags.bad = [
123 "base_PixelFlags_flag_edge",
124 "base_PixelFlags_flag_interpolatedCenter",
125 "base_PixelFlags_flag_saturatedCenter",
126 "base_PixelFlags_flag_crCenter",
127 "base_PixelFlags_flag_bad",
128 "base_PixelFlags_flag_interpolated",
129 "base_PixelFlags_flag_saturated",
130 ]
131 selector.signalToNoise.minimum = 200.0
132 selector.signalToNoise.maximum = None
133 selector.signalToNoise.fluxField = "base_PsfFlux_instFlux"
134 selector.signalToNoise.errField = "base_PsfFlux_instFluxErr"
135
136 def validate(self):
137 lsst.pex.config.Config.validate(self)
138 if self.sourceSelector.target.usesMatches:
139 raise lsst.pex.config.FieldValidationError(
140 MeasureApCorrConfig.sourceSelector,
141 self,
142 "Star selectors that require matches are not permitted.")
143
144
146 """Task to measure aperture correction.
147 """
148 ConfigClass = MeasureApCorrConfig
149 _DefaultName = "measureApCorr"
150
151 def __init__(self, schema, **kwargs):
152 """Construct a MeasureApCorrTask
153
154 For every name in lsst.meas.base.getApCorrNameSet():
155 - If the corresponding flux fields exist in the schema:
156 - Add a new field apcorr_{name}_used
157 - Add an entry to the self.toCorrect dict
158 - Otherwise silently skip the name
159 """
160 Task.__init__(self, **kwargs)
161 self.refFluxNames = _FluxNames(self.config.refFluxName, schema)
162 self.toCorrect = {} # dict of flux field name prefix: FluxKeys instance
163 for name in sorted(getApCorrNameSet()):
164 try:
165 self.toCorrect[name] = _FluxNames(name, schema)
166 except KeyError:
167 # if a field in the registry is missing, just ignore it.
168 pass
169 self.makeSubtask("sourceSelector")
170
171 def run(self, exposure, catalog):
172 """Measure aperture correction
173
174 Parameters
175 ----------
176 exposure : `lsst.afw.image.Exposure`
177 Exposure aperture corrections are being measured on. The
178 bounding box is retrieved from it, and it is passed to the
179 sourceSelector. The output aperture correction map is *not*
180 added to the exposure; this is left to the caller.
182 SourceCatalog containing measurements to be used to
183 compute aperture corrections.
184
185 Returns
186 -------
187 Struct : `lsst.pipe.base.Struct`
188 Contains the following:
189
190 ``apCorrMap``
191 aperture correction map (`lsst.afw.image.ApCorrMap`)
192 that contains two entries for each flux field:
193 - flux field (e.g. base_PsfFlux_instFlux): 2d model
194 - flux sigma field (e.g. base_PsfFlux_instFluxErr): 2d model of error
195 """
196 bbox = exposure.getBBox()
197 import lsstDebug
198 display = lsstDebug.Info(__name__).display
199 doPause = lsstDebug.Info(__name__).doPause
200
201 self.log.info("Measuring aperture corrections for %d flux fields", len(self.toCorrect))
202
203 # First, create a subset of the catalog that contains only selected stars
204 # with non-flagged reference fluxes.
205 selected = self.sourceSelector.run(catalog, exposure=exposure)
206
207 use = (
208 ~selected.sourceCat[self.refFluxNames.flagName]
209 & (np.isfinite(selected.sourceCat[self.refFluxNames.fluxName]))
210 )
211 goodRefCat = selected.sourceCat[use].copy()
212
213 apCorrMap = ApCorrMap()
214
215 # Outer loop over the fields we want to correct
216 for name, fluxNames in self.toCorrect.items():
217 # Create a more restricted subset with only the objects where the to-be-correct flux
218 # is not flagged.
219 fluxes = goodRefCat[fluxNames.fluxName]
220 with np.errstate(invalid="ignore"): # suppress NaN warnings.
221 isGood = (
222 (~goodRefCat[fluxNames.flagName])
223 & (np.isfinite(fluxes))
224 & (fluxes > 0.0)
225 )
226
227 # The 1 is the minimum number of ctrl.computeSize() when the order
228 # drops to 0 in both x and y.
229 if (isGood.sum() - 1) < self.config.minDegreesOfFreedom:
230 if name in self.config.allowFailure:
231 self.log.warning("Unable to measure aperture correction for '%s': "
232 "only %d sources, but require at least %d.",
233 name, isGood.sum(), self.config.minDegreesOfFreedom + 1)
234 continue
235 else:
236 msg = ("Unable to measure aperture correction for required algorithm '%s': "
237 "only %d sources, but require at least %d." %
238 (name, isGood.sum(), self.config.minDegreesOfFreedom + 1))
239 self.log.warning(msg)
240 raise MeasureApCorrError("Aperture correction failed on required algorithm.")
241
242 goodCat = goodRefCat[isGood].copy()
243
244 x = goodCat['slot_Centroid_x']
245 y = goodCat['slot_Centroid_y']
246 z = goodCat[self.refFluxNames.fluxName]/goodCat[fluxNames.fluxName]
247 ids = goodCat['id']
248
249 # We start with an initial fit that is the median offset; this
250 # works well in practice.
251 fitValues = np.median(z)
252
253 ctrl = self.config.fitConfig.makeControl()
254
255 allBad = False
256 for iteration in range(self.config.numIter):
257 resid = z - fitValues
258 # We add a small (epsilon) amount of floating-point slop because
259 # the median_abs_deviation may give a value that is just larger than 0
260 # even if given a completely flat residual field (as in tests).
261 apCorrErr = median_abs_deviation(resid, scale="normal") + 1e-7
262 keep = np.abs(resid) <= self.config.numSigmaClip * apCorrErr
263
264 self.log.debug("Removing %d sources as outliers.", len(resid) - keep.sum())
265
266 x = x[keep]
267 y = y[keep]
268 z = z[keep]
269 ids = ids[keep]
270
271 while (len(x) - ctrl.computeSize()) < self.config.minDegreesOfFreedom:
272 if ctrl.orderX > 0:
273 ctrl.orderX -= 1
274 else:
275 allBad = True
276 break
277 if ctrl.orderY > 0:
278 ctrl.orderY -= 1
279 else:
280 allBad = True
281 break
282
283 if allBad:
284 if name in self.config.allowFailure:
285 self.log.warning("Unable to measure aperture correction for '%s': "
286 "only %d sources remain, but require at least %d." %
287 (name, keep.sum(), self.config.minDegreesOfFreedom + 1))
288 break
289 else:
290 msg = ("Unable to measure aperture correction for required algorithm "
291 "'%s': only %d sources remain, but require at least %d." %
292 (name, keep.sum(), self.config.minDegreesOfFreedom + 1))
293 self.log.warning(msg)
294 raise MeasureApCorrError("Aperture correction failed on required algorithm.")
295
296 apCorrField = ChebyshevBoundedField.fit(bbox, x, y, z, ctrl)
297 fitValues = apCorrField.evaluate(x, y)
298
299 if allBad:
300 continue
301
302 self.log.info(
303 "Aperture correction for %s from %d stars: MAD %f, RMS %f",
304 name,
305 median_abs_deviation(fitValues - z, scale="normal"),
306 np.mean((fitValues - z)**2.)**0.5,
307 len(x),
308 )
309
310 if display:
311 plotApCorr(bbox, x, y, z, apCorrField, "%s, final" % (name,), doPause)
312
313 # Record which sources were used.
314 used = np.zeros(len(catalog), dtype=bool)
315 used[np.searchsorted(catalog['id'], ids)] = True
316 catalog[fluxNames.usedName] = used
317
318 # Save the result in the output map
319 # The error is constant spatially (we could imagine being
320 # more clever, but we're not yet sure if it's worth the effort).
321 # We save the errors as a 0th-order ChebyshevBoundedField
322 apCorrMap[fluxNames.fluxName] = apCorrField
323 apCorrMap[fluxNames.errName] = ChebyshevBoundedField(
324 bbox,
325 np.array([[apCorrErr]]),
326 )
327
328 return Struct(
329 apCorrMap=apCorrMap,
330 )
331
332
333def plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause):
334 """Plot aperture correction fit residuals
335
336 There are two subplots: residuals against x and y.
337
338 Intended for debugging.
339
340 Parameters
341 ----------
342 bbox : `lsst.geom.Box2I`
343 Bounding box (for bounds)
344 xx, yy : `numpy.ndarray`, (N)
345 x and y coordinates
346 zzMeasure : `float`
347 Measured value of the aperture correction
349 Fit aperture correction field
350 title : 'str'
351 Title for plot
352 doPause : `bool`
353 Pause to inspect the residuals plot? If
354 False, there will be a 4 second delay to
355 allow for inspection of the plot before
356 closing it and moving on.
357 """
358 import matplotlib.pyplot as plt
359
360 zzFit = field.evaluate(xx, yy)
361 residuals = zzMeasure - zzFit
362
363 fig, axes = plt.subplots(2, 1)
364
365 axes[0].scatter(xx, residuals, s=3, marker='o', lw=0, alpha=0.7)
366 axes[1].scatter(yy, residuals, s=3, marker='o', lw=0, alpha=0.7)
367 for ax in axes:
368 ax.set_ylabel("ApCorr Fit Residual")
369 ax.set_ylim(0.9*residuals.min(), 1.1*residuals.max())
370 axes[0].set_xlabel("x")
371 axes[0].set_xlim(bbox.getMinX(), bbox.getMaxX())
372 axes[1].set_xlabel("y")
373 axes[1].set_xlim(bbox.getMinY(), bbox.getMaxY())
374 plt.suptitle(title)
375
376 if not doPause:
377 try:
378 plt.pause(4)
379 plt.close()
380 except Exception:
381 print("%s: plt.pause() failed. Please close plots when done." % __name__)
382 plt.show()
383 else:
384 print("%s: Please close plots when done." % __name__)
385 plt.show()
plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause)