lsst.meas.algorithms g2edf3c4d4e+142cdb03a3
Loading...
Searching...
No Matches
measureApCorr.py
Go to the documentation of this file.
2# LSST Data Management System
3#
4# Copyright 2008-2017 AURA/LSST.
5#
6# This product includes software developed by the
7# LSST Project (http://www.lsst.org/).
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the LSST License Statement and
20# the GNU General Public License along with this program. If not,
21# see <https://www.lsstcorp.org/LegalNotices/>.
22#
23
24__all__ = ("MeasureApCorrConfig", "MeasureApCorrTask", "MeasureApCorrError")
25
26import numpy as np
27from scipy.stats import median_abs_deviation
28
29import lsst.pex.config
30from lsst.afw.image import ApCorrMap
31from lsst.afw.math import ChebyshevBoundedField, ChebyshevBoundedFieldConfig
32from lsst.pipe.base import Task, Struct
33from lsst.meas.base.apCorrRegistry import getApCorrNameSet
34
35from .sourceSelector import sourceSelectorRegistry
36
37
38class MeasureApCorrError(RuntimeError):
39 pass
40
41
43 """A collection of flux-related names for a given flux measurement algorithm.
44
45 Parameters
46 ----------
47 name : `str`
48 Name of flux measurement algorithm, e.g. ``base_PsfFlux``.
49 schema : `lsst.afw.table.Schema`
50 Catalog schema containing the flux field. The ``{name}_instFlux``,
51 ``{name}_instFluxErr``, ``{name}_flag`` fields are checked for
52 existence, and the ``apcorr_{name}_used`` field is added.
53
54 Raises
55 ------
56 KeyError if any of instFlux, instFluxErr, or flag fields is missing.
57 """
58 def __init__(self, name, schema):
59 self.fluxName = name + "_instFlux"
60 if self.fluxName not in schema:
61 raise KeyError("Could not find " + self.fluxName)
62 self.errName = name + "_instFluxErr"
63 if self.errName not in schema:
64 raise KeyError("Could not find " + self.errName)
65 self.flagName = name + "_flag"
66 if self.flagName not in schema:
67 raise KeyError("Cound not find " + self.flagName)
68 self.usedName = "apcorr_" + name + "_used"
69 schema.addField(self.usedName, type="Flag",
70 doc="Set if source was used in measuring aperture correction.")
71
72
73class MeasureApCorrConfig(lsst.pex.config.Config):
74 """Configuration for MeasureApCorrTask.
75 """
76 refFluxName = lsst.pex.config.Field(
77 doc="Field name prefix for the flux other measurements should be aperture corrected to match",
78 dtype=str,
79 default="slot_CalibFlux",
80 )
81 sourceSelector = sourceSelectorRegistry.makeField(
82 doc="Selector that sets the stars that aperture corrections will be measured from.",
83 default="science",
84 )
85 minDegreesOfFreedom = lsst.pex.config.RangeField(
86 doc="Minimum number of degrees of freedom (# of valid data points - # of parameters);"
87 " if this is exceeded, the order of the fit is decreased (in both dimensions), and"
88 " if we can't decrease it enough, we'll raise ValueError.",
89 dtype=int,
90 default=1,
91 min=1,
92 )
93 fitConfig = lsst.pex.config.ConfigField(
94 doc="Configuration used in fitting the aperture correction fields.",
95 dtype=ChebyshevBoundedFieldConfig,
96 )
97 numIter = lsst.pex.config.Field(
98 doc="Number of iterations for robust MAD sigma clipping.",
99 dtype=int,
100 default=4,
101 )
102 numSigmaClip = lsst.pex.config.Field(
103 doc="Number of robust MAD sigma to do clipping.",
104 dtype=float,
105 default=4.0,
106 )
107 allowFailure = lsst.pex.config.ListField(
108 doc="Allow these measurement algorithms to fail without an exception.",
109 dtype=str,
110 default=[],
111 )
112
113 def setDefaults(self):
114 selector = self.sourceSelector["science"]
115
116 selector.doFluxLimit = False
117 selector.doFlags = True
118 selector.doUnresolved = True
119 selector.doSignalToNoise = True
120 selector.doIsolated = False
121 selector.flags.good = []
122 selector.flags.bad = [
123 "base_PixelFlags_flag_edge",
124 "base_PixelFlags_flag_interpolatedCenter",
125 "base_PixelFlags_flag_saturatedCenter",
126 "base_PixelFlags_flag_crCenter",
127 "base_PixelFlags_flag_bad",
128 "base_PixelFlags_flag_interpolated",
129 "base_PixelFlags_flag_saturated",
130 ]
131 selector.signalToNoise.minimum = 200.0
132 selector.signalToNoise.maximum = None
133 selector.signalToNoise.fluxField = "base_PsfFlux_instFlux"
134 selector.signalToNoise.errField = "base_PsfFlux_instFluxErr"
135
136 def validate(self):
137 lsst.pex.config.Config.validate(self)
138 if self.sourceSelector.target.usesMatches:
139 raise lsst.pex.config.FieldValidationError(
140 MeasureApCorrConfig.sourceSelector,
141 self,
142 "Star selectors that require matches are not permitted.")
143
144
146 """Task to measure aperture correction.
147 """
148 ConfigClass = MeasureApCorrConfig
149 _DefaultName = "measureApCorr"
150
151 def __init__(self, schema, **kwargs):
152 """Construct a MeasureApCorrTask
153
154 For every name in lsst.meas.base.getApCorrNameSet():
155 - If the corresponding flux fields exist in the schema:
156 - Add a new field apcorr_{name}_used
157 - Add an entry to the self.toCorrect dict
158 - Otherwise silently skip the name
159 """
160 Task.__init__(self, **kwargs)
161 self.refFluxNames = _FluxNames(self.config.refFluxName, schema)
162 self.toCorrect = {} # dict of flux field name prefix: FluxKeys instance
163 for name in sorted(getApCorrNameSet()):
164 try:
165 self.toCorrect[name] = _FluxNames(name, schema)
166 except KeyError:
167 # if a field in the registry is missing, just ignore it.
168 pass
169 self.makeSubtask("sourceSelector")
170
171 def run(self, exposure, catalog):
172 """Measure aperture correction
173
174 Parameters
175 ----------
176 exposure : `lsst.afw.image.Exposure`
177 Exposure aperture corrections are being measured on. The
178 bounding box is retrieved from it, and it is passed to the
179 sourceSelector. The output aperture correction map is *not*
180 added to the exposure; this is left to the caller.
182 SourceCatalog containing measurements to be used to
183 compute aperture corrections.
184
185 Returns
186 -------
187 Struct : `lsst.pipe.base.Struct`
188 Contains the following:
189
190 ``apCorrMap``
191 aperture correction map (`lsst.afw.image.ApCorrMap`)
192 that contains two entries for each flux field:
193 - flux field (e.g. base_PsfFlux_instFlux): 2d model
194 - flux sigma field (e.g. base_PsfFlux_instFluxErr): 2d model of error
195 """
196 bbox = exposure.getBBox()
197 import lsstDebug
198 display = lsstDebug.Info(__name__).display
199 doPause = lsstDebug.Info(__name__).doPause
200
201 self.log.info("Measuring aperture corrections for %d flux fields", len(self.toCorrect))
202
203 # First, create a subset of the catalog that contains only selected stars
204 # with non-flagged reference fluxes.
205 selected = self.sourceSelector.run(catalog, exposure=exposure)
206
207 use = (
208 ~selected.sourceCat[self.refFluxNames.flagName]
209 & (np.isfinite(selected.sourceCat[self.refFluxNames.fluxName]))
210 )
211 goodRefCat = selected.sourceCat[use].copy()
212
213 apCorrMap = ApCorrMap()
214
215 # Outer loop over the fields we want to correct
216 for name, fluxNames in self.toCorrect.items():
217 # Create a more restricted subset with only the objects where the to-be-correct flux
218 # is not flagged.
219 fluxes = goodRefCat[fluxNames.fluxName]
220 with np.errstate(invalid="ignore"): # suppress NaN warnings.
221 isGood = (
222 (~goodRefCat[fluxNames.flagName])
223 & (np.isfinite(fluxes))
224 & (fluxes > 0.0)
225 )
226
227 # The 1 is the minimum number of ctrl.computeSize() when the order
228 # drops to 0 in both x and y.
229 if (isGood.sum() - 1) < self.config.minDegreesOfFreedom:
230 if name in self.config.allowFailure:
231 self.log.warning("Unable to measure aperture correction for '%s': "
232 "only %d sources, but require at least %d.",
233 name, isGood.sum(), self.config.minDegreesOfFreedom + 1)
234 continue
235 else:
236 msg = ("Unable to measure aperture correction for required algorithm '%s': "
237 "only %d sources, but require at least %d." %
238 (name, isGood.sum(), self.config.minDegreesOfFreedom + 1))
239 self.log.warning(msg)
240 raise MeasureApCorrError("Aperture correction failed on required algorithm.")
241
242 goodCat = goodRefCat[isGood].copy()
243
244 x = goodCat['slot_Centroid_x']
245 y = goodCat['slot_Centroid_y']
246 z = goodCat[self.refFluxNames.fluxName]/goodCat[fluxNames.fluxName]
247 ids = goodCat['id']
248
249 # We start with an initial fit that is the median offset; this
250 # works well in practice.
251 fitValues = np.median(z)
252
253 ctrl = self.config.fitConfig.makeControl()
254
255 allBad = False
256 for iteration in range(self.config.numIter):
257 resid = z - fitValues
258 # We add a small (epsilon) amount of floating-point slop because
259 # the median_abs_deviation may give a value that is just larger than 0
260 # even if given a completely flat residual field (as in tests).
261 apCorrErr = median_abs_deviation(resid, scale="normal") + 1e-7
262 keep = np.abs(resid) <= self.config.numSigmaClip * apCorrErr
263
264 self.log.debug("Removing %d sources as outliers.", len(resid) - keep.sum())
265
266 x = x[keep]
267 y = y[keep]
268 z = z[keep]
269 ids = ids[keep]
270
271 while (len(x) - ctrl.computeSize()) < self.config.minDegreesOfFreedom:
272 if ctrl.orderX > 0:
273 ctrl.orderX -= 1
274 else:
275 allBad = True
276 break
277 if ctrl.orderY > 0:
278 ctrl.orderY -= 1
279 else:
280 allBad = True
281 break
282
283 if allBad:
284 if name in self.config.allowFailure:
285 self.log.warning("Unable to measure aperture correction for '%s': "
286 "only %d sources remain, but require at least %d." %
287 (name, keep.sum(), self.config.minDegreesOfFreedom + 1))
288 break
289 else:
290 raise RuntimeError("Unable to measure aperture correction for required algorithm "
291 "'%s': only %d sources remain, but require at least %d." %
292 (name, keep.sum(), self.config.minDegreesOfFreedom + 1))
293
294 apCorrField = ChebyshevBoundedField.fit(bbox, x, y, z, ctrl)
295 fitValues = apCorrField.evaluate(x, y)
296
297 if allBad:
298 continue
299
300 self.log.info(
301 "Aperture correction for %s from %d stars: MAD %f, RMS %f",
302 name,
303 median_abs_deviation(fitValues - z, scale="normal"),
304 np.mean((fitValues - z)**2.)**0.5,
305 len(x),
306 )
307
308 if display:
309 plotApCorr(bbox, x, y, z, apCorrField, "%s, final" % (name,), doPause)
310
311 # Record which sources were used.
312 used = np.zeros(len(catalog), dtype=bool)
313 used[np.searchsorted(catalog['id'], ids)] = True
314 catalog[fluxNames.usedName] = used
315
316 # Save the result in the output map
317 # The error is constant spatially (we could imagine being
318 # more clever, but we're not yet sure if it's worth the effort).
319 # We save the errors as a 0th-order ChebyshevBoundedField
320 apCorrMap[fluxNames.fluxName] = apCorrField
321 apCorrMap[fluxNames.errName] = ChebyshevBoundedField(
322 bbox,
323 np.array([[apCorrErr]]),
324 )
325
326 return Struct(
327 apCorrMap=apCorrMap,
328 )
329
330
331def plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause):
332 """Plot aperture correction fit residuals
333
334 There are two subplots: residuals against x and y.
335
336 Intended for debugging.
337
338 Parameters
339 ----------
340 bbox : `lsst.geom.Box2I`
341 Bounding box (for bounds)
342 xx, yy : `numpy.ndarray`, (N)
343 x and y coordinates
344 zzMeasure : `float`
345 Measured value of the aperture correction
347 Fit aperture correction field
348 title : 'str'
349 Title for plot
350 doPause : `bool`
351 Pause to inspect the residuals plot? If
352 False, there will be a 4 second delay to
353 allow for inspection of the plot before
354 closing it and moving on.
355 """
356 import matplotlib.pyplot as plt
357
358 zzFit = field.evaluate(xx, yy)
359 residuals = zzMeasure - zzFit
360
361 fig, axes = plt.subplots(2, 1)
362
363 axes[0].scatter(xx, residuals, s=3, marker='o', lw=0, alpha=0.7)
364 axes[1].scatter(yy, residuals, s=3, marker='o', lw=0, alpha=0.7)
365 for ax in axes:
366 ax.set_ylabel("ApCorr Fit Residual")
367 ax.set_ylim(0.9*residuals.min(), 1.1*residuals.max())
368 axes[0].set_xlabel("x")
369 axes[0].set_xlim(bbox.getMinX(), bbox.getMaxX())
370 axes[1].set_xlabel("y")
371 axes[1].set_xlim(bbox.getMinY(), bbox.getMaxY())
372 plt.suptitle(title)
373
374 if not doPause:
375 try:
376 plt.pause(4)
377 plt.close()
378 except Exception:
379 print("%s: plt.pause() failed. Please close plots when done." % __name__)
380 plt.show()
381 else:
382 print("%s: Please close plots when done." % __name__)
383 plt.show()
def plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause)