Coverage for python/lsst/meas/algorithms/measureApCorr.py: 15%
152 statements
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-16 11:09 +0000
« prev ^ index » next coverage.py v7.4.0, created at 2024-01-16 11:09 +0000
1#
2# LSST Data Management System
3#
4# Copyright 2008-2017 AURA/LSST.
5#
6# This product includes software developed by the
7# LSST Project (http://www.lsst.org/).
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the LSST License Statement and
20# the GNU General Public License along with this program. If not,
21# see <https://www.lsstcorp.org/LegalNotices/>.
22#
24__all__ = ("MeasureApCorrConfig", "MeasureApCorrTask", "MeasureApCorrError")
26import numpy as np
27from scipy.stats import median_abs_deviation
29import lsst.pex.config
30from lsst.afw.image import ApCorrMap
31from lsst.afw.math import ChebyshevBoundedField, ChebyshevBoundedFieldConfig
32from lsst.pipe.base import Task, Struct
33from lsst.meas.base.apCorrRegistry import getApCorrNameSet
35from .sourceSelector import sourceSelectorRegistry
38class MeasureApCorrError(RuntimeError):
39 pass
42class _FluxNames:
43 """A collection of flux-related names for a given flux measurement algorithm.
45 Parameters
46 ----------
47 name : `str`
48 Name of flux measurement algorithm, e.g. ``base_PsfFlux``.
49 schema : `lsst.afw.table.Schema`
50 Catalog schema containing the flux field. The ``{name}_instFlux``,
51 ``{name}_instFluxErr``, ``{name}_flag`` fields are checked for
52 existence, and the ``apcorr_{name}_used`` field is added.
54 Raises
55 ------
56 KeyError if any of instFlux, instFluxErr, or flag fields is missing.
57 """
58 def __init__(self, name, schema):
59 self.fluxName = name + "_instFlux"
60 if self.fluxName not in schema:
61 raise KeyError("Could not find " + self.fluxName)
62 self.errName = name + "_instFluxErr"
63 if self.errName not in schema:
64 raise KeyError("Could not find " + self.errName)
65 self.flagName = name + "_flag"
66 if self.flagName not in schema:
67 raise KeyError("Cound not find " + self.flagName)
68 self.usedName = "apcorr_" + name + "_used"
69 schema.addField(self.usedName, type="Flag",
70 doc="Set if source was used in measuring aperture correction.")
73class MeasureApCorrConfig(lsst.pex.config.Config):
74 """Configuration for MeasureApCorrTask.
75 """
76 refFluxName = lsst.pex.config.Field(
77 doc="Field name prefix for the flux other measurements should be aperture corrected to match",
78 dtype=str,
79 default="slot_CalibFlux",
80 )
81 sourceSelector = sourceSelectorRegistry.makeField(
82 doc="Selector that sets the stars that aperture corrections will be measured from.",
83 default="science",
84 )
85 minDegreesOfFreedom = lsst.pex.config.RangeField(
86 doc="Minimum number of degrees of freedom (# of valid data points - # of parameters);"
87 " if this is exceeded, the order of the fit is decreased (in both dimensions), and"
88 " if we can't decrease it enough, we'll raise ValueError.",
89 dtype=int,
90 default=1,
91 min=1,
92 )
93 fitConfig = lsst.pex.config.ConfigField(
94 doc="Configuration used in fitting the aperture correction fields.",
95 dtype=ChebyshevBoundedFieldConfig,
96 )
97 numIter = lsst.pex.config.Field(
98 doc="Number of iterations for robust MAD sigma clipping.",
99 dtype=int,
100 default=4,
101 )
102 numSigmaClip = lsst.pex.config.Field(
103 doc="Number of robust MAD sigma to do clipping.",
104 dtype=float,
105 default=4.0,
106 )
107 allowFailure = lsst.pex.config.ListField(
108 doc="Allow these measurement algorithms to fail without an exception.",
109 dtype=str,
110 default=[],
111 )
113 def setDefaults(self):
114 selector = self.sourceSelector["science"]
116 selector.doFlags = True
117 selector.doUnresolved = True
118 selector.doSignalToNoise = True
119 selector.doIsolated = False
120 selector.flags.good = []
121 selector.flags.bad = [
122 "base_PixelFlags_flag_edge",
123 "base_PixelFlags_flag_interpolatedCenter",
124 "base_PixelFlags_flag_saturatedCenter",
125 "base_PixelFlags_flag_crCenter",
126 "base_PixelFlags_flag_bad",
127 "base_PixelFlags_flag_interpolated",
128 "base_PixelFlags_flag_saturated",
129 ]
130 selector.signalToNoise.minimum = 200.0
131 selector.signalToNoise.maximum = None
132 selector.signalToNoise.fluxField = "base_PsfFlux_instFlux"
133 selector.signalToNoise.errField = "base_PsfFlux_instFluxErr"
135 def validate(self):
136 lsst.pex.config.Config.validate(self)
137 if self.sourceSelector.target.usesMatches:
138 raise lsst.pex.config.FieldValidationError(
139 MeasureApCorrConfig.sourceSelector,
140 self,
141 "Star selectors that require matches are not permitted.")
144class MeasureApCorrTask(Task):
145 """Task to measure aperture correction.
146 """
147 ConfigClass = MeasureApCorrConfig
148 _DefaultName = "measureApCorr"
150 def __init__(self, schema, **kwargs):
151 """Construct a MeasureApCorrTask
153 For every name in lsst.meas.base.getApCorrNameSet():
154 - If the corresponding flux fields exist in the schema:
155 - Add a new field apcorr_{name}_used
156 - Add an entry to the self.toCorrect dict
157 - Otherwise silently skip the name
158 """
159 Task.__init__(self, **kwargs)
160 self.refFluxNames = _FluxNames(self.config.refFluxName, schema)
161 self.toCorrect = {} # dict of flux field name prefix: FluxKeys instance
162 for name in sorted(getApCorrNameSet()):
163 try:
164 self.toCorrect[name] = _FluxNames(name, schema)
165 except KeyError:
166 # if a field in the registry is missing, just ignore it.
167 pass
168 self.makeSubtask("sourceSelector")
170 def run(self, exposure, catalog):
171 """Measure aperture correction
173 Parameters
174 ----------
175 exposure : `lsst.afw.image.Exposure`
176 Exposure aperture corrections are being measured on. The
177 bounding box is retrieved from it, and it is passed to the
178 sourceSelector. The output aperture correction map is *not*
179 added to the exposure; this is left to the caller.
180 catalog : `lsst.afw.table.SourceCatalog`
181 SourceCatalog containing measurements to be used to
182 compute aperture corrections.
184 Returns
185 -------
186 Struct : `lsst.pipe.base.Struct`
187 Contains the following:
189 ``apCorrMap``
190 aperture correction map (`lsst.afw.image.ApCorrMap`)
191 that contains two entries for each flux field:
192 - flux field (e.g. base_PsfFlux_instFlux): 2d model
193 - flux sigma field (e.g. base_PsfFlux_instFluxErr): 2d model of error
194 """
195 bbox = exposure.getBBox()
196 import lsstDebug
197 display = lsstDebug.Info(__name__).display
198 doPause = lsstDebug.Info(__name__).doPause
200 self.log.info("Measuring aperture corrections for %d flux fields", len(self.toCorrect))
202 # First, create a subset of the catalog that contains only selected stars
203 # with non-flagged reference fluxes.
204 selected = self.sourceSelector.run(catalog, exposure=exposure)
206 use = (
207 ~selected.sourceCat[self.refFluxNames.flagName]
208 & (np.isfinite(selected.sourceCat[self.refFluxNames.fluxName]))
209 )
210 goodRefCat = selected.sourceCat[use].copy()
212 apCorrMap = ApCorrMap()
214 # Outer loop over the fields we want to correct
215 for name, fluxNames in self.toCorrect.items():
216 # Create a more restricted subset with only the objects where the to-be-correct flux
217 # is not flagged.
218 fluxes = goodRefCat[fluxNames.fluxName]
219 with np.errstate(invalid="ignore"): # suppress NaN warnings.
220 isGood = (
221 (~goodRefCat[fluxNames.flagName])
222 & (np.isfinite(fluxes))
223 & (fluxes > 0.0)
224 )
226 # The 1 is the minimum number of ctrl.computeSize() when the order
227 # drops to 0 in both x and y.
228 if (isGood.sum() - 1) < self.config.minDegreesOfFreedom:
229 if name in self.config.allowFailure:
230 self.log.warning("Unable to measure aperture correction for '%s': "
231 "only %d sources, but require at least %d.",
232 name, isGood.sum(), self.config.minDegreesOfFreedom + 1)
233 continue
234 else:
235 msg = ("Unable to measure aperture correction for required algorithm '%s': "
236 "only %d sources, but require at least %d." %
237 (name, isGood.sum(), self.config.minDegreesOfFreedom + 1))
238 self.log.warning(msg)
239 raise MeasureApCorrError("Aperture correction failed on required algorithm.")
241 goodCat = goodRefCat[isGood].copy()
243 x = goodCat['slot_Centroid_x']
244 y = goodCat['slot_Centroid_y']
245 z = goodCat[self.refFluxNames.fluxName]/goodCat[fluxNames.fluxName]
246 ids = goodCat['id']
248 # We start with an initial fit that is the median offset; this
249 # works well in practice.
250 fitValues = np.median(z)
252 ctrl = self.config.fitConfig.makeControl()
254 allBad = False
255 for iteration in range(self.config.numIter):
256 resid = z - fitValues
257 # We add a small (epsilon) amount of floating-point slop because
258 # the median_abs_deviation may give a value that is just larger than 0
259 # even if given a completely flat residual field (as in tests).
260 apCorrErr = median_abs_deviation(resid, scale="normal") + 1e-7
261 keep = np.abs(resid) <= self.config.numSigmaClip * apCorrErr
263 self.log.debug("Removing %d sources as outliers.", len(resid) - keep.sum())
265 x = x[keep]
266 y = y[keep]
267 z = z[keep]
268 ids = ids[keep]
270 while (len(x) - ctrl.computeSize()) < self.config.minDegreesOfFreedom:
271 if ctrl.orderX > 0:
272 ctrl.orderX -= 1
273 else:
274 allBad = True
275 break
276 if ctrl.orderY > 0:
277 ctrl.orderY -= 1
278 else:
279 allBad = True
280 break
282 if allBad:
283 if name in self.config.allowFailure:
284 self.log.warning("Unable to measure aperture correction for '%s': "
285 "only %d sources remain, but require at least %d." %
286 (name, keep.sum(), self.config.minDegreesOfFreedom + 1))
287 break
288 else:
289 msg = ("Unable to measure aperture correction for required algorithm "
290 "'%s': only %d sources remain, but require at least %d." %
291 (name, keep.sum(), self.config.minDegreesOfFreedom + 1))
292 self.log.warning(msg)
293 raise MeasureApCorrError("Aperture correction failed on required algorithm.")
295 apCorrField = ChebyshevBoundedField.fit(bbox, x, y, z, ctrl)
296 fitValues = apCorrField.evaluate(x, y)
298 if allBad:
299 continue
301 self.log.info(
302 "Aperture correction for %s from %d stars: MAD %f, RMS %f",
303 name,
304 median_abs_deviation(fitValues - z, scale="normal"),
305 np.mean((fitValues - z)**2.)**0.5,
306 len(x),
307 )
309 if display:
310 plotApCorr(bbox, x, y, z, apCorrField, "%s, final" % (name,), doPause)
312 # Record which sources were used.
313 used = np.zeros(len(catalog), dtype=bool)
314 used[np.searchsorted(catalog['id'], ids)] = True
315 catalog[fluxNames.usedName] = used
317 # Save the result in the output map
318 # The error is constant spatially (we could imagine being
319 # more clever, but we're not yet sure if it's worth the effort).
320 # We save the errors as a 0th-order ChebyshevBoundedField
321 apCorrMap[fluxNames.fluxName] = apCorrField
322 apCorrMap[fluxNames.errName] = ChebyshevBoundedField(
323 bbox,
324 np.array([[apCorrErr]]),
325 )
327 return Struct(
328 apCorrMap=apCorrMap,
329 )
332def plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause):
333 """Plot aperture correction fit residuals
335 There are two subplots: residuals against x and y.
337 Intended for debugging.
339 Parameters
340 ----------
341 bbox : `lsst.geom.Box2I`
342 Bounding box (for bounds)
343 xx, yy : `numpy.ndarray`, (N)
344 x and y coordinates
345 zzMeasure : `float`
346 Measured value of the aperture correction
347 field : `lsst.afw.math.ChebyshevBoundedField`
348 Fit aperture correction field
349 title : 'str'
350 Title for plot
351 doPause : `bool`
352 Pause to inspect the residuals plot? If
353 False, there will be a 4 second delay to
354 allow for inspection of the plot before
355 closing it and moving on.
356 """
357 import matplotlib.pyplot as plt
359 zzFit = field.evaluate(xx, yy)
360 residuals = zzMeasure - zzFit
362 fig, axes = plt.subplots(2, 1)
364 axes[0].scatter(xx, residuals, s=3, marker='o', lw=0, alpha=0.7)
365 axes[1].scatter(yy, residuals, s=3, marker='o', lw=0, alpha=0.7)
366 for ax in axes:
367 ax.set_ylabel("ApCorr Fit Residual")
368 ax.set_ylim(0.9*residuals.min(), 1.1*residuals.max())
369 axes[0].set_xlabel("x")
370 axes[0].set_xlim(bbox.getMinX(), bbox.getMaxX())
371 axes[1].set_xlabel("y")
372 axes[1].set_xlim(bbox.getMinY(), bbox.getMaxY())
373 plt.suptitle(title)
375 if not doPause:
376 try:
377 plt.pause(4)
378 plt.close()
379 except Exception:
380 print("%s: plt.pause() failed. Please close plots when done." % __name__)
381 plt.show()
382 else:
383 print("%s: Please close plots when done." % __name__)
384 plt.show()