lsst.meas.algorithms ga883e5f241+b78ff77604
Loading...
Searching...
No Matches
measureApCorr.py
Go to the documentation of this file.
2# LSST Data Management System
3#
4# Copyright 2008-2017 AURA/LSST.
5#
6# This product includes software developed by the
7# LSST Project (http://www.lsst.org/).
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the LSST License Statement and
20# the GNU General Public License along with this program. If not,
21# see <https://www.lsstcorp.org/LegalNotices/>.
22#
23
24__all__ = ("MeasureApCorrConfig", "MeasureApCorrTask", "MeasureApCorrError")
25
26import numpy as np
27from scipy.stats import median_abs_deviation
28
29import lsst.pex.config
30from lsst.afw.image import ApCorrMap
31from lsst.afw.math import ChebyshevBoundedField, ChebyshevBoundedFieldConfig
32from lsst.pipe.base import Task, Struct
33from lsst.meas.base.apCorrRegistry import getApCorrNameSet
34
35from .sourceSelector import sourceSelectorRegistry
36
37
38class MeasureApCorrError(RuntimeError):
39 pass
40
41
43 """A collection of flux-related names for a given flux measurement algorithm.
44
45 Parameters
46 ----------
47 name : `str`
48 Name of flux measurement algorithm, e.g. ``base_PsfFlux``.
49 schema : `lsst.afw.table.Schema`
50 Catalog schema containing the flux field. The ``{name}_instFlux``,
51 ``{name}_instFluxErr``, ``{name}_flag`` fields are checked for
52 existence, and the ``apcorr_{name}_used`` field is added.
53
54 Raises
55 ------
56 KeyError if any of instFlux, instFluxErr, or flag fields is missing.
57 """
58 def __init__(self, name, schema):
59 self.fluxName = name + "_instFlux"
60 if self.fluxName not in schema:
61 raise KeyError("Could not find " + self.fluxName)
62 self.errName = name + "_instFluxErr"
63 if self.errName not in schema:
64 raise KeyError("Could not find " + self.errName)
65 self.flagName = name + "_flag"
66 if self.flagName not in schema:
67 raise KeyError("Cound not find " + self.flagName)
68 self.usedName = "apcorr_" + name + "_used"
69 schema.addField(self.usedName, type="Flag",
70 doc="Set if source was used in measuring aperture correction.")
71
72
73class MeasureApCorrConfig(lsst.pex.config.Config):
74 """Configuration for MeasureApCorrTask.
75 """
76 refFluxName = lsst.pex.config.Field(
77 doc="Field name prefix for the flux other measurements should be aperture corrected to match",
78 dtype=str,
79 default="slot_CalibFlux",
80 )
81 sourceSelector = sourceSelectorRegistry.makeField(
82 doc="Selector that sets the stars that aperture corrections will be measured from.",
83 default="science",
84 )
85 minDegreesOfFreedom = lsst.pex.config.RangeField(
86 doc="Minimum number of degrees of freedom (# of valid data points - # of parameters);"
87 " if this is exceeded, the order of the fit is decreased (in both dimensions), and"
88 " if we can't decrease it enough, we'll raise ValueError.",
89 dtype=int,
90 default=1,
91 min=1,
92 )
93 fitConfig = lsst.pex.config.ConfigField(
94 doc="Configuration used in fitting the aperture correction fields.",
95 dtype=ChebyshevBoundedFieldConfig,
96 )
97 numIter = lsst.pex.config.Field(
98 doc="Number of iterations for robust MAD sigma clipping.",
99 dtype=int,
100 default=4,
101 )
102 numSigmaClip = lsst.pex.config.Field(
103 doc="Number of robust MAD sigma to do clipping.",
104 dtype=float,
105 default=4.0,
106 )
107 allowFailure = lsst.pex.config.ListField(
108 doc="Allow these measurement algorithms to fail without an exception.",
109 dtype=str,
110 default=[],
111 )
112
113 def setDefaults(self):
114 selector = self.sourceSelector["science"]
115
116 selector.doFlags = True
117 selector.doUnresolved = True
118 selector.doSignalToNoise = True
119 selector.doIsolated = False
120 selector.flags.good = []
121 selector.flags.bad = [
122 "base_PixelFlags_flag_edge",
123 "base_PixelFlags_flag_interpolatedCenter",
124 "base_PixelFlags_flag_saturatedCenter",
125 "base_PixelFlags_flag_crCenter",
126 "base_PixelFlags_flag_bad",
127 "base_PixelFlags_flag_interpolated",
128 "base_PixelFlags_flag_saturated",
129 ]
130 selector.signalToNoise.minimum = 200.0
131 selector.signalToNoise.maximum = None
132 selector.signalToNoise.fluxField = "base_PsfFlux_instFlux"
133 selector.signalToNoise.errField = "base_PsfFlux_instFluxErr"
134
135 def validate(self):
136 lsst.pex.config.Config.validate(self)
137 if self.sourceSelector.target.usesMatches:
138 raise lsst.pex.config.FieldValidationError(
139 MeasureApCorrConfig.sourceSelector,
140 self,
141 "Star selectors that require matches are not permitted.")
142
143
145 """Task to measure aperture correction.
146 """
147 ConfigClass = MeasureApCorrConfig
148 _DefaultName = "measureApCorr"
149
150 def __init__(self, schema, **kwargs):
151 """Construct a MeasureApCorrTask
152
153 For every name in lsst.meas.base.getApCorrNameSet():
154 - If the corresponding flux fields exist in the schema:
155 - Add a new field apcorr_{name}_used
156 - Add an entry to the self.toCorrect dict
157 - Otherwise silently skip the name
158 """
159 Task.__init__(self, **kwargs)
160 self.refFluxNames = _FluxNames(self.config.refFluxName, schema)
161 self.toCorrect = {} # dict of flux field name prefix: FluxKeys instance
162 for name in sorted(getApCorrNameSet()):
163 try:
164 self.toCorrect[name] = _FluxNames(name, schema)
165 except KeyError:
166 # if a field in the registry is missing, just ignore it.
167 pass
168 self.makeSubtask("sourceSelector")
169
170 def run(self, exposure, catalog):
171 """Measure aperture correction
172
173 Parameters
174 ----------
175 exposure : `lsst.afw.image.Exposure`
176 Exposure aperture corrections are being measured on. The
177 bounding box is retrieved from it, and it is passed to the
178 sourceSelector. The output aperture correction map is *not*
179 added to the exposure; this is left to the caller.
180 catalog : `lsst.afw.table.SourceCatalog`
181 SourceCatalog containing measurements to be used to
182 compute aperture corrections.
183
184 Returns
185 -------
186 Struct : `lsst.pipe.base.Struct`
187 Contains the following:
188
189 ``apCorrMap``
190 aperture correction map (`lsst.afw.image.ApCorrMap`)
191 that contains two entries for each flux field:
192 - flux field (e.g. base_PsfFlux_instFlux): 2d model
193 - flux sigma field (e.g. base_PsfFlux_instFluxErr): 2d model of error
194 """
195 bbox = exposure.getBBox()
196 import lsstDebug
197 display = lsstDebug.Info(__name__).display
198 doPause = lsstDebug.Info(__name__).doPause
199
200 self.log.info("Measuring aperture corrections for %d flux fields", len(self.toCorrect))
201
202 # First, create a subset of the catalog that contains only selected stars
203 # with non-flagged reference fluxes.
204 selected = self.sourceSelector.run(catalog, exposure=exposure)
205
206 use = (
207 ~selected.sourceCat[self.refFluxNames.flagName]
208 & (np.isfinite(selected.sourceCat[self.refFluxNames.fluxName]))
209 )
210 goodRefCat = selected.sourceCat[use].copy()
211
212 apCorrMap = ApCorrMap()
213
214 # Outer loop over the fields we want to correct
215 for name, fluxNames in self.toCorrect.items():
216 # Create a more restricted subset with only the objects where the to-be-correct flux
217 # is not flagged.
218 fluxes = goodRefCat[fluxNames.fluxName]
219 with np.errstate(invalid="ignore"): # suppress NaN warnings.
220 isGood = (
221 (~goodRefCat[fluxNames.flagName])
222 & (np.isfinite(fluxes))
223 & (fluxes > 0.0)
224 )
225
226 # The 1 is the minimum number of ctrl.computeSize() when the order
227 # drops to 0 in both x and y.
228 if (isGood.sum() - 1) < self.config.minDegreesOfFreedom:
229 if name in self.config.allowFailure:
230 self.log.warning("Unable to measure aperture correction for '%s': "
231 "only %d sources, but require at least %d.",
232 name, isGood.sum(), self.config.minDegreesOfFreedom + 1)
233 continue
234 else:
235 msg = ("Unable to measure aperture correction for required algorithm '%s': "
236 "only %d sources, but require at least %d." %
237 (name, isGood.sum(), self.config.minDegreesOfFreedom + 1))
238 self.log.warning(msg)
239 raise MeasureApCorrError("Aperture correction failed on required algorithm.")
240
241 goodCat = goodRefCat[isGood].copy()
242
243 x = goodCat['slot_Centroid_x']
244 y = goodCat['slot_Centroid_y']
245 z = goodCat[self.refFluxNames.fluxName]/goodCat[fluxNames.fluxName]
246 ids = goodCat['id']
247
248 # We start with an initial fit that is the median offset; this
249 # works well in practice.
250 fitValues = np.median(z)
251
252 ctrl = self.config.fitConfig.makeControl()
253
254 allBad = False
255 for iteration in range(self.config.numIter):
256 resid = z - fitValues
257 # We add a small (epsilon) amount of floating-point slop because
258 # the median_abs_deviation may give a value that is just larger than 0
259 # even if given a completely flat residual field (as in tests).
260 apCorrErr = median_abs_deviation(resid, scale="normal") + 1e-7
261 keep = np.abs(resid) <= self.config.numSigmaClip * apCorrErr
262
263 self.log.debug("Removing %d sources as outliers.", len(resid) - keep.sum())
264
265 x = x[keep]
266 y = y[keep]
267 z = z[keep]
268 ids = ids[keep]
269
270 while (len(x) - ctrl.computeSize()) < self.config.minDegreesOfFreedom:
271 if ctrl.orderX > 0:
272 ctrl.orderX -= 1
273 else:
274 allBad = True
275 break
276 if ctrl.orderY > 0:
277 ctrl.orderY -= 1
278 else:
279 allBad = True
280 break
281
282 if allBad:
283 if name in self.config.allowFailure:
284 self.log.warning("Unable to measure aperture correction for '%s': "
285 "only %d sources remain, but require at least %d." %
286 (name, keep.sum(), self.config.minDegreesOfFreedom + 1))
287 break
288 else:
289 msg = ("Unable to measure aperture correction for required algorithm "
290 "'%s': only %d sources remain, but require at least %d." %
291 (name, keep.sum(), self.config.minDegreesOfFreedom + 1))
292 self.log.warning(msg)
293 raise MeasureApCorrError("Aperture correction failed on required algorithm.")
294
295 apCorrField = ChebyshevBoundedField.fit(bbox, x, y, z, ctrl)
296 fitValues = apCorrField.evaluate(x, y)
297
298 if allBad:
299 continue
300
301 self.log.info(
302 "Aperture correction for %s from %d stars: MAD %f, RMS %f",
303 name,
304 median_abs_deviation(fitValues - z, scale="normal"),
305 np.mean((fitValues - z)**2.)**0.5,
306 len(x),
307 )
308
309 if display:
310 plotApCorr(bbox, x, y, z, apCorrField, "%s, final" % (name,), doPause)
311
312 # Record which sources were used.
313 used = np.zeros(len(catalog), dtype=bool)
314 used[np.searchsorted(catalog['id'], ids)] = True
315 catalog[fluxNames.usedName] = used
316
317 # Save the result in the output map
318 # The error is constant spatially (we could imagine being
319 # more clever, but we're not yet sure if it's worth the effort).
320 # We save the errors as a 0th-order ChebyshevBoundedField
321 apCorrMap[fluxNames.fluxName] = apCorrField
322 apCorrMap[fluxNames.errName] = ChebyshevBoundedField(
323 bbox,
324 np.array([[apCorrErr]]),
325 )
326
327 return Struct(
328 apCorrMap=apCorrMap,
329 )
330
331
332def plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause):
333 """Plot aperture correction fit residuals
334
335 There are two subplots: residuals against x and y.
336
337 Intended for debugging.
338
339 Parameters
340 ----------
341 bbox : `lsst.geom.Box2I`
342 Bounding box (for bounds)
343 xx, yy : `numpy.ndarray`, (N)
344 x and y coordinates
345 zzMeasure : `float`
346 Measured value of the aperture correction
347 field : `lsst.afw.math.ChebyshevBoundedField`
348 Fit aperture correction field
349 title : 'str'
350 Title for plot
351 doPause : `bool`
352 Pause to inspect the residuals plot? If
353 False, there will be a 4 second delay to
354 allow for inspection of the plot before
355 closing it and moving on.
356 """
357 import matplotlib.pyplot as plt
358
359 zzFit = field.evaluate(xx, yy)
360 residuals = zzMeasure - zzFit
361
362 fig, axes = plt.subplots(2, 1)
363
364 axes[0].scatter(xx, residuals, s=3, marker='o', lw=0, alpha=0.7)
365 axes[1].scatter(yy, residuals, s=3, marker='o', lw=0, alpha=0.7)
366 for ax in axes:
367 ax.set_ylabel("ApCorr Fit Residual")
368 ax.set_ylim(0.9*residuals.min(), 1.1*residuals.max())
369 axes[0].set_xlabel("x")
370 axes[0].set_xlim(bbox.getMinX(), bbox.getMaxX())
371 axes[1].set_xlabel("y")
372 axes[1].set_xlim(bbox.getMinY(), bbox.getMaxY())
373 plt.suptitle(title)
374
375 if not doPause:
376 try:
377 plt.pause(4)
378 plt.close()
379 except Exception:
380 print("%s: plt.pause() failed. Please close plots when done." % __name__)
381 plt.show()
382 else:
383 print("%s: Please close plots when done." % __name__)
384 plt.show()
plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause)