Coverage for python/lsst/meas/algorithms/measureApCorr.py: 15%
153 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-15 03:09 -0700
« prev ^ index » next coverage.py v7.2.7, created at 2023-06-15 03:09 -0700
1#
2# LSST Data Management System
3#
4# Copyright 2008-2017 AURA/LSST.
5#
6# This product includes software developed by the
7# LSST Project (http://www.lsst.org/).
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the LSST License Statement and
20# the GNU General Public License along with this program. If not,
21# see <https://www.lsstcorp.org/LegalNotices/>.
22#
24__all__ = ("MeasureApCorrConfig", "MeasureApCorrTask", "MeasureApCorrError")
26import numpy as np
27from scipy.stats import median_abs_deviation
29import lsst.pex.config
30from lsst.afw.image import ApCorrMap
31from lsst.afw.math import ChebyshevBoundedField, ChebyshevBoundedFieldConfig
32from lsst.pipe.base import Task, Struct
33from lsst.meas.base.apCorrRegistry import getApCorrNameSet
35from .sourceSelector import sourceSelectorRegistry
38class MeasureApCorrError(RuntimeError):
39 pass
42class _FluxNames:
43 """A collection of flux-related names for a given flux measurement algorithm.
45 Parameters
46 ----------
47 name : `str`
48 Name of flux measurement algorithm, e.g. ``base_PsfFlux``.
49 schema : `lsst.afw.table.Schema`
50 Catalog schema containing the flux field. The ``{name}_instFlux``,
51 ``{name}_instFluxErr``, ``{name}_flag`` fields are checked for
52 existence, and the ``apcorr_{name}_used`` field is added.
54 Raises
55 ------
56 KeyError if any of instFlux, instFluxErr, or flag fields is missing.
57 """
58 def __init__(self, name, schema):
59 self.fluxName = name + "_instFlux"
60 if self.fluxName not in schema:
61 raise KeyError("Could not find " + self.fluxName)
62 self.errName = name + "_instFluxErr"
63 if self.errName not in schema:
64 raise KeyError("Could not find " + self.errName)
65 self.flagName = name + "_flag"
66 if self.flagName not in schema:
67 raise KeyError("Cound not find " + self.flagName)
68 self.usedName = "apcorr_" + name + "_used"
69 schema.addField(self.usedName, type="Flag",
70 doc="Set if source was used in measuring aperture correction.")
73class MeasureApCorrConfig(lsst.pex.config.Config):
74 """Configuration for MeasureApCorrTask.
75 """
76 refFluxName = lsst.pex.config.Field(
77 doc="Field name prefix for the flux other measurements should be aperture corrected to match",
78 dtype=str,
79 default="slot_CalibFlux",
80 )
81 sourceSelector = sourceSelectorRegistry.makeField(
82 doc="Selector that sets the stars that aperture corrections will be measured from.",
83 default="science",
84 )
85 minDegreesOfFreedom = lsst.pex.config.RangeField(
86 doc="Minimum number of degrees of freedom (# of valid data points - # of parameters);"
87 " if this is exceeded, the order of the fit is decreased (in both dimensions), and"
88 " if we can't decrease it enough, we'll raise ValueError.",
89 dtype=int,
90 default=1,
91 min=1,
92 )
93 fitConfig = lsst.pex.config.ConfigField(
94 doc="Configuration used in fitting the aperture correction fields.",
95 dtype=ChebyshevBoundedFieldConfig,
96 )
97 numIter = lsst.pex.config.Field(
98 doc="Number of iterations for robust MAD sigma clipping.",
99 dtype=int,
100 default=4,
101 )
102 numSigmaClip = lsst.pex.config.Field(
103 doc="Number of robust MAD sigma to do clipping.",
104 dtype=float,
105 default=4.0,
106 )
107 allowFailure = lsst.pex.config.ListField(
108 doc="Allow these measurement algorithms to fail without an exception.",
109 dtype=str,
110 default=[],
111 )
113 def setDefaults(self):
114 selector = self.sourceSelector["science"]
116 selector.doFluxLimit = False
117 selector.doFlags = True
118 selector.doUnresolved = True
119 selector.doSignalToNoise = True
120 selector.doIsolated = False
121 selector.flags.good = []
122 selector.flags.bad = [
123 "base_PixelFlags_flag_edge",
124 "base_PixelFlags_flag_interpolatedCenter",
125 "base_PixelFlags_flag_saturatedCenter",
126 "base_PixelFlags_flag_crCenter",
127 "base_PixelFlags_flag_bad",
128 "base_PixelFlags_flag_interpolated",
129 "base_PixelFlags_flag_saturated",
130 ]
131 selector.signalToNoise.minimum = 200.0
132 selector.signalToNoise.maximum = None
133 selector.signalToNoise.fluxField = "base_PsfFlux_instFlux"
134 selector.signalToNoise.errField = "base_PsfFlux_instFluxErr"
136 def validate(self):
137 lsst.pex.config.Config.validate(self)
138 if self.sourceSelector.target.usesMatches:
139 raise lsst.pex.config.FieldValidationError(
140 MeasureApCorrConfig.sourceSelector,
141 self,
142 "Star selectors that require matches are not permitted.")
145class MeasureApCorrTask(Task):
146 """Task to measure aperture correction.
147 """
148 ConfigClass = MeasureApCorrConfig
149 _DefaultName = "measureApCorr"
151 def __init__(self, schema, **kwargs):
152 """Construct a MeasureApCorrTask
154 For every name in lsst.meas.base.getApCorrNameSet():
155 - If the corresponding flux fields exist in the schema:
156 - Add a new field apcorr_{name}_used
157 - Add an entry to the self.toCorrect dict
158 - Otherwise silently skip the name
159 """
160 Task.__init__(self, **kwargs)
161 self.refFluxNames = _FluxNames(self.config.refFluxName, schema)
162 self.toCorrect = {} # dict of flux field name prefix: FluxKeys instance
163 for name in sorted(getApCorrNameSet()):
164 try:
165 self.toCorrect[name] = _FluxNames(name, schema)
166 except KeyError:
167 # if a field in the registry is missing, just ignore it.
168 pass
169 self.makeSubtask("sourceSelector")
171 def run(self, exposure, catalog):
172 """Measure aperture correction
174 Parameters
175 ----------
176 exposure : `lsst.afw.image.Exposure`
177 Exposure aperture corrections are being measured on. The
178 bounding box is retrieved from it, and it is passed to the
179 sourceSelector. The output aperture correction map is *not*
180 added to the exposure; this is left to the caller.
181 catalog : `lsst.afw.table.SourceCatalog`
182 SourceCatalog containing measurements to be used to
183 compute aperture corrections.
185 Returns
186 -------
187 Struct : `lsst.pipe.base.Struct`
188 Contains the following:
190 ``apCorrMap``
191 aperture correction map (`lsst.afw.image.ApCorrMap`)
192 that contains two entries for each flux field:
193 - flux field (e.g. base_PsfFlux_instFlux): 2d model
194 - flux sigma field (e.g. base_PsfFlux_instFluxErr): 2d model of error
195 """
196 bbox = exposure.getBBox()
197 import lsstDebug
198 display = lsstDebug.Info(__name__).display
199 doPause = lsstDebug.Info(__name__).doPause
201 self.log.info("Measuring aperture corrections for %d flux fields", len(self.toCorrect))
203 # First, create a subset of the catalog that contains only selected stars
204 # with non-flagged reference fluxes.
205 selected = self.sourceSelector.run(catalog, exposure=exposure)
207 use = (
208 ~selected.sourceCat[self.refFluxNames.flagName]
209 & (np.isfinite(selected.sourceCat[self.refFluxNames.fluxName]))
210 )
211 goodRefCat = selected.sourceCat[use].copy()
213 apCorrMap = ApCorrMap()
215 # Outer loop over the fields we want to correct
216 for name, fluxNames in self.toCorrect.items():
217 # Create a more restricted subset with only the objects where the to-be-correct flux
218 # is not flagged.
219 fluxes = goodRefCat[fluxNames.fluxName]
220 with np.errstate(invalid="ignore"): # suppress NaN warnings.
221 isGood = (
222 (~goodRefCat[fluxNames.flagName])
223 & (np.isfinite(fluxes))
224 & (fluxes > 0.0)
225 )
227 # The 1 is the minimum number of ctrl.computeSize() when the order
228 # drops to 0 in both x and y.
229 if (isGood.sum() - 1) < self.config.minDegreesOfFreedom:
230 if name in self.config.allowFailure:
231 self.log.warning("Unable to measure aperture correction for '%s': "
232 "only %d sources, but require at least %d.",
233 name, isGood.sum(), self.config.minDegreesOfFreedom + 1)
234 continue
235 else:
236 msg = ("Unable to measure aperture correction for required algorithm '%s': "
237 "only %d sources, but require at least %d." %
238 (name, isGood.sum(), self.config.minDegreesOfFreedom + 1))
239 self.log.warning(msg)
240 raise MeasureApCorrError("Aperture correction failed on required algorithm.")
242 goodCat = goodRefCat[isGood].copy()
244 x = goodCat['slot_Centroid_x']
245 y = goodCat['slot_Centroid_y']
246 z = goodCat[self.refFluxNames.fluxName]/goodCat[fluxNames.fluxName]
247 ids = goodCat['id']
249 # We start with an initial fit that is the median offset; this
250 # works well in practice.
251 fitValues = np.median(z)
253 ctrl = self.config.fitConfig.makeControl()
255 allBad = False
256 for iteration in range(self.config.numIter):
257 resid = z - fitValues
258 # We add a small (epsilon) amount of floating-point slop because
259 # the median_abs_deviation may give a value that is just larger than 0
260 # even if given a completely flat residual field (as in tests).
261 apCorrErr = median_abs_deviation(resid, scale="normal") + 1e-7
262 keep = np.abs(resid) <= self.config.numSigmaClip * apCorrErr
264 self.log.debug("Removing %d sources as outliers.", len(resid) - keep.sum())
266 x = x[keep]
267 y = y[keep]
268 z = z[keep]
269 ids = ids[keep]
271 while (len(x) - ctrl.computeSize()) < self.config.minDegreesOfFreedom:
272 if ctrl.orderX > 0:
273 ctrl.orderX -= 1
274 else:
275 allBad = True
276 break
277 if ctrl.orderY > 0:
278 ctrl.orderY -= 1
279 else:
280 allBad = True
281 break
283 if allBad:
284 if name in self.config.allowFailure:
285 self.log.warning("Unable to measure aperture correction for '%s': "
286 "only %d sources remain, but require at least %d." %
287 (name, keep.sum(), self.config.minDegreesOfFreedom + 1))
288 break
289 else:
290 msg = ("Unable to measure aperture correction for required algorithm "
291 "'%s': only %d sources remain, but require at least %d." %
292 (name, keep.sum(), self.config.minDegreesOfFreedom + 1))
293 self.log.warning(msg)
294 raise MeasureApCorrError("Aperture correction failed on required algorithm.")
296 apCorrField = ChebyshevBoundedField.fit(bbox, x, y, z, ctrl)
297 fitValues = apCorrField.evaluate(x, y)
299 if allBad:
300 continue
302 self.log.info(
303 "Aperture correction for %s from %d stars: MAD %f, RMS %f",
304 name,
305 median_abs_deviation(fitValues - z, scale="normal"),
306 np.mean((fitValues - z)**2.)**0.5,
307 len(x),
308 )
310 if display:
311 plotApCorr(bbox, x, y, z, apCorrField, "%s, final" % (name,), doPause)
313 # Record which sources were used.
314 used = np.zeros(len(catalog), dtype=bool)
315 used[np.searchsorted(catalog['id'], ids)] = True
316 catalog[fluxNames.usedName] = used
318 # Save the result in the output map
319 # The error is constant spatially (we could imagine being
320 # more clever, but we're not yet sure if it's worth the effort).
321 # We save the errors as a 0th-order ChebyshevBoundedField
322 apCorrMap[fluxNames.fluxName] = apCorrField
323 apCorrMap[fluxNames.errName] = ChebyshevBoundedField(
324 bbox,
325 np.array([[apCorrErr]]),
326 )
328 return Struct(
329 apCorrMap=apCorrMap,
330 )
333def plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause):
334 """Plot aperture correction fit residuals
336 There are two subplots: residuals against x and y.
338 Intended for debugging.
340 Parameters
341 ----------
342 bbox : `lsst.geom.Box2I`
343 Bounding box (for bounds)
344 xx, yy : `numpy.ndarray`, (N)
345 x and y coordinates
346 zzMeasure : `float`
347 Measured value of the aperture correction
348 field : `lsst.afw.math.ChebyshevBoundedField`
349 Fit aperture correction field
350 title : 'str'
351 Title for plot
352 doPause : `bool`
353 Pause to inspect the residuals plot? If
354 False, there will be a 4 second delay to
355 allow for inspection of the plot before
356 closing it and moving on.
357 """
358 import matplotlib.pyplot as plt
360 zzFit = field.evaluate(xx, yy)
361 residuals = zzMeasure - zzFit
363 fig, axes = plt.subplots(2, 1)
365 axes[0].scatter(xx, residuals, s=3, marker='o', lw=0, alpha=0.7)
366 axes[1].scatter(yy, residuals, s=3, marker='o', lw=0, alpha=0.7)
367 for ax in axes:
368 ax.set_ylabel("ApCorr Fit Residual")
369 ax.set_ylim(0.9*residuals.min(), 1.1*residuals.max())
370 axes[0].set_xlabel("x")
371 axes[0].set_xlim(bbox.getMinX(), bbox.getMaxX())
372 axes[1].set_xlabel("y")
373 axes[1].set_xlim(bbox.getMinY(), bbox.getMaxY())
374 plt.suptitle(title)
376 if not doPause:
377 try:
378 plt.pause(4)
379 plt.close()
380 except Exception:
381 print("%s: plt.pause() failed. Please close plots when done." % __name__)
382 plt.show()
383 else:
384 print("%s: Please close plots when done." % __name__)
385 plt.show()