Coverage for python/lsst/meas/algorithms/measureApCorr.py: 15%
147 statements
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-01 02:20 -0700
« prev ^ index » next coverage.py v6.5.0, created at 2023-04-01 02:20 -0700
1#
2# LSST Data Management System
3#
4# Copyright 2008-2017 AURA/LSST.
5#
6# This product includes software developed by the
7# LSST Project (http://www.lsst.org/).
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the LSST License Statement and
20# the GNU General Public License along with this program. If not,
21# see <https://www.lsstcorp.org/LegalNotices/>.
22#
24__all__ = ("MeasureApCorrConfig", "MeasureApCorrTask")
26import numpy as np
27from scipy.stats import median_abs_deviation
29import lsst.pex.config
30from lsst.afw.image import ApCorrMap
31from lsst.afw.math import ChebyshevBoundedField, ChebyshevBoundedFieldConfig
32from lsst.pipe.base import Task, Struct
33from lsst.meas.base.apCorrRegistry import getApCorrNameSet
35from .sourceSelector import sourceSelectorRegistry
38class _FluxNames:
39 """A collection of flux-related names for a given flux measurement algorithm.
41 Parameters
42 ----------
43 name : `str`
44 Name of flux measurement algorithm, e.g. ``base_PsfFlux``.
45 schema : `lsst.afw.table.Schema`
46 Catalog schema containing the flux field. The ``{name}_instFlux``,
47 ``{name}_instFluxErr``, ``{name}_flag`` fields are checked for
48 existence, and the ``apcorr_{name}_used`` field is added.
50 Raises
51 ------
52 KeyError if any of instFlux, instFluxErr, or flag fields is missing.
53 """
54 def __init__(self, name, schema):
55 self.fluxName = name + "_instFlux"
56 if self.fluxName not in schema:
57 raise KeyError("Could not find " + self.fluxName)
58 self.errName = name + "_instFluxErr"
59 if self.errName not in schema:
60 raise KeyError("Could not find " + self.errName)
61 self.flagName = name + "_flag"
62 if self.flagName not in schema:
63 raise KeyError("Cound not find " + self.flagName)
64 self.usedName = "apcorr_" + name + "_used"
65 schema.addField(self.usedName, type="Flag",
66 doc="Set if source was used in measuring aperture correction.")
69class MeasureApCorrConfig(lsst.pex.config.Config):
70 """Configuration for MeasureApCorrTask.
71 """
72 refFluxName = lsst.pex.config.Field(
73 doc="Field name prefix for the flux other measurements should be aperture corrected to match",
74 dtype=str,
75 default="slot_CalibFlux",
76 )
77 sourceSelector = sourceSelectorRegistry.makeField(
78 doc="Selector that sets the stars that aperture corrections will be measured from.",
79 default="science",
80 )
81 minDegreesOfFreedom = lsst.pex.config.RangeField(
82 doc="Minimum number of degrees of freedom (# of valid data points - # of parameters);"
83 " if this is exceeded, the order of the fit is decreased (in both dimensions), and"
84 " if we can't decrease it enough, we'll raise ValueError.",
85 dtype=int,
86 default=1,
87 min=1,
88 )
89 fitConfig = lsst.pex.config.ConfigField(
90 doc="Configuration used in fitting the aperture correction fields.",
91 dtype=ChebyshevBoundedFieldConfig,
92 )
93 numIter = lsst.pex.config.Field(
94 doc="Number of iterations for robust MAD sigma clipping.",
95 dtype=int,
96 default=4,
97 )
98 numSigmaClip = lsst.pex.config.Field(
99 doc="Number of robust MAD sigma to do clipping.",
100 dtype=float,
101 default=4.0,
102 )
103 allowFailure = lsst.pex.config.ListField(
104 doc="Allow these measurement algorithms to fail without an exception.",
105 dtype=str,
106 default=[],
107 )
109 def setDefaults(self):
110 selector = self.sourceSelector["science"]
112 selector.doFluxLimit = False
113 selector.doFlags = True
114 selector.doUnresolved = True
115 selector.doSignalToNoise = True
116 selector.doIsolated = False
117 selector.flags.good = []
118 selector.flags.bad = [
119 "base_PixelFlags_flag_edge",
120 "base_PixelFlags_flag_interpolatedCenter",
121 "base_PixelFlags_flag_saturatedCenter",
122 "base_PixelFlags_flag_crCenter",
123 "base_PixelFlags_flag_bad",
124 "base_PixelFlags_flag_interpolated",
125 "base_PixelFlags_flag_saturated",
126 ]
127 selector.signalToNoise.minimum = 200.0
128 selector.signalToNoise.maximum = None
129 selector.signalToNoise.fluxField = "base_PsfFlux_instFlux"
130 selector.signalToNoise.errField = "base_PsfFlux_instFluxErr"
132 def validate(self):
133 lsst.pex.config.Config.validate(self)
134 if self.sourceSelector.target.usesMatches:
135 raise lsst.pex.config.FieldValidationError(
136 MeasureApCorrConfig.sourceSelector,
137 self,
138 "Star selectors that require matches are not permitted.")
141class MeasureApCorrTask(Task):
142 """Task to measure aperture correction.
143 """
144 ConfigClass = MeasureApCorrConfig
145 _DefaultName = "measureApCorr"
147 def __init__(self, schema, **kwargs):
148 """Construct a MeasureApCorrTask
150 For every name in lsst.meas.base.getApCorrNameSet():
151 - If the corresponding flux fields exist in the schema:
152 - Add a new field apcorr_{name}_used
153 - Add an entry to the self.toCorrect dict
154 - Otherwise silently skip the name
155 """
156 Task.__init__(self, **kwargs)
157 self.refFluxNames = _FluxNames(self.config.refFluxName, schema)
158 self.toCorrect = {} # dict of flux field name prefix: FluxKeys instance
159 for name in sorted(getApCorrNameSet()):
160 try:
161 self.toCorrect[name] = _FluxNames(name, schema)
162 except KeyError:
163 # if a field in the registry is missing, just ignore it.
164 pass
165 self.makeSubtask("sourceSelector")
167 def run(self, exposure, catalog):
168 """Measure aperture correction
170 Parameters
171 ----------
172 exposure : `lsst.afw.image.Exposure`
173 Exposure aperture corrections are being measured on. The
174 bounding box is retrieved from it, and it is passed to the
175 sourceSelector. The output aperture correction map is *not*
176 added to the exposure; this is left to the caller.
177 catalog : `lsst.afw.table.SourceCatalog`
178 SourceCatalog containing measurements to be used to
179 compute aperture corrections.
181 Returns
182 -------
183 Struct : `lsst.pipe.base.Struct`
184 Contains the following:
186 ``apCorrMap``
187 aperture correction map (`lsst.afw.image.ApCorrMap`)
188 that contains two entries for each flux field:
189 - flux field (e.g. base_PsfFlux_instFlux): 2d model
190 - flux sigma field (e.g. base_PsfFlux_instFluxErr): 2d model of error
191 """
192 bbox = exposure.getBBox()
193 import lsstDebug
194 display = lsstDebug.Info(__name__).display
195 doPause = lsstDebug.Info(__name__).doPause
197 self.log.info("Measuring aperture corrections for %d flux fields", len(self.toCorrect))
199 # First, create a subset of the catalog that contains only selected stars
200 # with non-flagged reference fluxes.
201 selected = self.sourceSelector.run(catalog, exposure=exposure)
203 use = (
204 ~selected.sourceCat[self.refFluxNames.flagName]
205 & (np.isfinite(selected.sourceCat[self.refFluxNames.fluxName]))
206 )
207 goodRefCat = selected.sourceCat[use].copy()
209 apCorrMap = ApCorrMap()
211 # Outer loop over the fields we want to correct
212 for name, fluxNames in self.toCorrect.items():
213 # Create a more restricted subset with only the objects where the to-be-correct flux
214 # is not flagged.
215 fluxes = goodRefCat[fluxNames.fluxName]
216 with np.errstate(invalid="ignore"): # suppress NaN warnings.
217 isGood = (
218 (~goodRefCat[fluxNames.flagName])
219 & (np.isfinite(fluxes))
220 & (fluxes > 0.0)
221 )
223 # The 1 is the minimum number of ctrl.computeSize() when the order
224 # drops to 0 in both x and y.
225 if (isGood.sum() - 1) < self.config.minDegreesOfFreedom:
226 if name in self.config.allowFailure:
227 self.log.warning("Unable to measure aperture correction for '%s': "
228 "only %d sources, but require at least %d." %
229 (name, isGood.sum(), self.config.minDegreesOfFreedom + 1))
230 continue
231 else:
232 raise RuntimeError("Unable to measure aperture correction for required algorithm '%s': "
233 "only %d sources, but require at least %d." %
234 (name, isGood.sum(), self.config.minDegreesOfFreedom + 1))
236 goodCat = goodRefCat[isGood].copy()
238 x = goodCat['slot_Centroid_x']
239 y = goodCat['slot_Centroid_y']
240 z = goodCat[self.refFluxNames.fluxName]/goodCat[fluxNames.fluxName]
241 ids = goodCat['id']
243 # We start with an initial fit that is the median offset; this
244 # works well in practice.
245 fitValues = np.median(z)
247 ctrl = self.config.fitConfig.makeControl()
249 allBad = False
250 for iteration in range(self.config.numIter):
251 resid = z - fitValues
252 # We add a small (epsilon) amount of floating-point slop because
253 # the median_abs_deviation may give a value that is just larger than 0
254 # even if given a completely flat residual field (as in tests).
255 apCorrErr = median_abs_deviation(resid, scale="normal") + 1e-7
256 keep = np.abs(resid) <= self.config.numSigmaClip * apCorrErr
258 self.log.debug("Removing %d sources as outliers.", len(resid) - keep.sum())
260 x = x[keep]
261 y = y[keep]
262 z = z[keep]
263 ids = ids[keep]
265 while (len(x) - ctrl.computeSize()) < self.config.minDegreesOfFreedom:
266 if ctrl.orderX > 0:
267 ctrl.orderX -= 1
268 else:
269 allBad = True
270 break
271 if ctrl.orderY > 0:
272 ctrl.orderY -= 1
273 else:
274 allBad = True
275 break
277 if allBad:
278 if name in self.config.allowFailure:
279 self.log.warning("Unable to measure aperture correction for '%s': "
280 "only %d sources remain, but require at least %d." %
281 (name, keep.sum(), self.config.minDegreesOfFreedom + 1))
282 break
283 else:
284 raise RuntimeError("Unable to measure aperture correction for required algorithm "
285 "'%s': only %d sources remain, but require at least %d." %
286 (name, keep.sum(), self.config.minDegreesOfFreedom + 1))
288 apCorrField = ChebyshevBoundedField.fit(bbox, x, y, z, ctrl)
289 fitValues = apCorrField.evaluate(x, y)
291 if allBad:
292 continue
294 self.log.info(
295 "Aperture correction for %s from %d stars: MAD %f, RMS %f",
296 name,
297 median_abs_deviation(fitValues - z, scale="normal"),
298 np.mean((fitValues - z)**2.)**0.5,
299 len(x),
300 )
302 if display:
303 plotApCorr(bbox, x, y, z, apCorrField, "%s, final" % (name,), doPause)
305 # Record which sources were used.
306 used = np.zeros(len(catalog), dtype=bool)
307 used[np.searchsorted(catalog['id'], ids)] = True
308 catalog[fluxNames.usedName] = used
310 # Save the result in the output map
311 # The error is constant spatially (we could imagine being
312 # more clever, but we're not yet sure if it's worth the effort).
313 # We save the errors as a 0th-order ChebyshevBoundedField
314 apCorrMap[fluxNames.fluxName] = apCorrField
315 apCorrMap[fluxNames.errName] = ChebyshevBoundedField(
316 bbox,
317 np.array([[apCorrErr]]),
318 )
320 return Struct(
321 apCorrMap=apCorrMap,
322 )
325def plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause):
326 """Plot aperture correction fit residuals
328 There are two subplots: residuals against x and y.
330 Intended for debugging.
332 Parameters
333 ----------
334 bbox : `lsst.geom.Box2I`
335 Bounding box (for bounds)
336 xx, yy : `numpy.ndarray`, (N)
337 x and y coordinates
338 zzMeasure : `float`
339 Measured value of the aperture correction
340 field : `lsst.afw.math.ChebyshevBoundedField`
341 Fit aperture correction field
342 title : 'str'
343 Title for plot
344 doPause : `bool`
345 Pause to inspect the residuals plot? If
346 False, there will be a 4 second delay to
347 allow for inspection of the plot before
348 closing it and moving on.
349 """
350 import matplotlib.pyplot as plt
352 zzFit = field.evaluate(xx, yy)
353 residuals = zzMeasure - zzFit
355 fig, axes = plt.subplots(2, 1)
357 axes[0].scatter(xx, residuals, s=3, marker='o', lw=0, alpha=0.7)
358 axes[1].scatter(yy, residuals, s=3, marker='o', lw=0, alpha=0.7)
359 for ax in axes:
360 ax.set_ylabel("ApCorr Fit Residual")
361 ax.set_ylim(0.9*residuals.min(), 1.1*residuals.max())
362 axes[0].set_xlabel("x")
363 axes[0].set_xlim(bbox.getMinX(), bbox.getMaxX())
364 axes[1].set_xlabel("y")
365 axes[1].set_xlim(bbox.getMinY(), bbox.getMaxY())
366 plt.suptitle(title)
368 if not doPause:
369 try:
370 plt.pause(4)
371 plt.close()
372 except Exception:
373 print("%s: plt.pause() failed. Please close plots when done." % __name__)
374 plt.show()
375 else:
376 print("%s: Please close plots when done." % __name__)
377 plt.show()