lsst.meas.algorithms  15.0-13-g0ee414d5+1
measureApCorr.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 
24 __all__ = ("MeasureApCorrConfig", "MeasureApCorrTask")
25 
26 import numpy
27 
28 import lsst.pex.config
29 from lsst.afw.image import ApCorrMap
30 from lsst.afw.math import ChebyshevBoundedField, ChebyshevBoundedFieldConfig
31 from lsst.pipe.base import Task, Struct
32 from lsst.meas.base.apCorrRegistry import getApCorrNameSet
33 
34 from .sourceSelector import sourceSelectorRegistry
35 
36 
37 class FluxKeys:
38  """A collection of keys for a given flux measurement algorithm
39  """
40  __slots__ = ("flux", "err", "flag", "used") # prevent accidentally adding fields
41 
42  def __init__(self, name, schema):
43  """Construct a FluxKeys
44 
45  @parma[in] name name of flux measurement algorithm, e.g. "base_PsfFlux"
46  @param[in,out] schema catalog schema containing the flux field
47  read: {name}_flux, {name}_fluxSigma, {name}_flag
48  added: apcorr_{name}_used
49  """
50  self.flux = schema.find(name + "_flux").key
51  self.err = schema.find(name + "_fluxSigma").key
52  self.flag = schema.find(name + "_flag").key
53  self.used = schema.addField("apcorr_" + name + "_used", type="Flag",
54  doc="set if source was used in measuring aperture correction")
55 
56 # The following block adds links to these tasks from the Task Documentation page.
57 
63 
64 
65 class MeasureApCorrConfig(lsst.pex.config.Config):
66  """!Configuration for MeasureApCorrTask
67  """
68  refFluxName = lsst.pex.config.Field(
69  doc="Field name prefix for the flux other measurements should be aperture corrected to match",
70  dtype=str,
71  default="slot_CalibFlux",
72  )
73  sourceSelector = sourceSelectorRegistry.makeField(
74  doc="Selector that sets the stars that aperture corrections will be measured from.",
75  default="flagged",
76  )
77  minDegreesOfFreedom = lsst.pex.config.RangeField(
78  doc="Minimum number of degrees of freedom (# of valid data points - # of parameters);" +
79  " if this is exceeded, the order of the fit is decreased (in both dimensions), and" +
80  " if we can't decrease it enough, we'll raise ValueError.",
81  dtype=int,
82  default=1,
83  min=1,
84  )
85  fitConfig = lsst.pex.config.ConfigField(
86  doc="Configuration used in fitting the aperture correction fields",
87  dtype=ChebyshevBoundedFieldConfig,
88  )
89  numIter = lsst.pex.config.Field(
90  doc="Number of iterations for sigma clipping",
91  dtype=int,
92  default=4,
93  )
94  numSigmaClip = lsst.pex.config.Field(
95  doc="Number of standard devisations to clip at",
96  dtype=float,
97  default=3.0,
98  )
99  allowFailure = lsst.pex.config.ListField(
100  doc="Allow these measurement algorithms to fail without an exception",
101  dtype=str,
102  default=[],
103  )
104 
105  def validate(self):
106  lsst.pex.config.Config.validate(self)
107  if self.sourceSelector.target.usesMatches:
108  raise lsst.pex.config.FieldValidationError(
109  "Star selectors that require matches are not permitted"
110  )
111 
112 
113 class MeasureApCorrTask(Task):
114  """!Task to measure aperture correction
115 
116  @section measAlg_MeasureApCorrTask_Contents Contents
117 
118  - @ref measAlg_MeasureApCorrTask_Purpose
119  - @ref measAlg_MeasureApCorrTask_Config
120  - @ref measAlg_MeasureApCorrTask_Debug
121 
122  @section measAlg_MeasureApCorrTask_Purpose Description
123 
124  @copybrief MeasureApCorrTask
125 
126  This task measures aperture correction for the flux fields returned by
127  lsst.meas.base.getApCorrNameSet()
128 
129  The main method is @ref MeasureApCorrTask.run "run".
130 
131  @section measAlg_MeasureApCorrTask_Config Configuration parameters
132 
133  See @ref MeasureApCorrConfig
134 
135  @section measAlg_MeasureApCorrTask_Debug Debug variables
136 
137  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a flag
138  `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug for more about `debug.py`.
139 
140  MeasureApCorrTask has a debug dictionary containing a single boolean key:
141  <dl>
142  <dt>display
143  <dd>If True: will show plots as aperture corrections are fitted
144  </dl>
145 
146  For example, put something like:
147  @code{.py}
148  import lsstDebug
149  def DebugInfo(name):
150  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
151  if name == "lsst.meas.algorithms.measureApCorr":
152  di.display = dict(
153  unsubtracted = 1,
154  subtracted = 2,
155  background = 3,
156  )
157 
158  return di
159 
160  lsstDebug.Info = DebugInfo
161  @endcode
162  into your `debug.py` file and run your command-line task with the `--debug` flag (or `import debug`).
163  """
164  ConfigClass = MeasureApCorrConfig
165  _DefaultName = "measureApCorr"
166 
167  def __init__(self, schema, **kwds):
168  """!Construct a MeasureApCorrTask
169 
170  For every name in lsst.meas.base.getApCorrNameSet():
171  - If the corresponding flux fields exist in the schema:
172  - Add a new field apcorr_{name}_used
173  - Add an entry to the self.toCorrect dict
174  - Otherwise silently skip the name
175  """
176  Task.__init__(self, **kwds)
177  self.refFluxKeys = FluxKeys(self.config.refFluxName, schema)
178  self.toCorrect = {} # dict of flux field name prefix: FluxKeys instance
179  for name in getApCorrNameSet():
180  try:
181  self.toCorrect[name] = FluxKeys(name, schema)
182  except KeyError:
183  # if a field in the registry is missing, just ignore it.
184  pass
185  self.makeSubtask("sourceSelector")
186 
187  def run(self, exposure, catalog):
188  """!Measure aperture correction
189 
190  @param[in] exposure Exposure aperture corrections are being measured
191  on. The bounding box is retrieved from it, and
192  it is passed to the sourceSelector.
193  The output aperture correction map is *not*
194  added to the exposure; this is left to the
195  caller.
196 
197  @param[in] catalog SourceCatalog containing measurements to be used
198  to compute aperturecorrections.
199 
200  @return an lsst.pipe.base.Struct containing:
201  - apCorrMap: an aperture correction map (lsst.afw.image.ApCorrMap) that contains two entries
202  for each flux field:
203  - flux field (e.g. base_PsfFlux_flux): 2d model
204  - flux sigma field (e.g. base_PsfFlux_fluxSigma): 2d model of error
205  """
206  bbox = exposure.getBBox()
207  import lsstDebug
208  display = lsstDebug.Info(__name__).display
209 
210  self.log.info("Measuring aperture corrections for %d flux fields" % (len(self.toCorrect),))
211  # First, create a subset of the catalog that contains only selected stars
212  # with non-flagged reference fluxes.
213  subset1 = [record for record in self.sourceSelector.run(catalog, exposure=exposure).sourceCat
214  if (not record.get(self.refFluxKeys.flag) and
215  numpy.isfinite(record.get(self.refFluxKeys.flux)))]
216 
217  apCorrMap = ApCorrMap()
218 
219  # Outer loop over the fields we want to correct
220  for name, keys in self.toCorrect.items():
221  fluxName = name + "_flux"
222  fluxSigmaName = name + "_fluxSigma"
223 
224  # Create a more restricted subset with only the objects where the to-be-correct flux
225  # is not flagged.
226  fluxes = numpy.fromiter((record.get(keys.flux) for record in subset1), float)
227  with numpy.errstate(invalid="ignore"): # suppress NAN warnings
228  isGood = numpy.logical_and.reduce([
229  numpy.fromiter((not record.get(keys.flag) for record in subset1), bool),
230  numpy.isfinite(fluxes),
231  fluxes > 0.0,
232  ])
233  subset2 = [record for record, good in zip(subset1, isGood) if good]
234 
235  # Check that we have enough data points that we have at least the minimum of degrees of
236  # freedom specified in the config.
237  if len(subset2) - 1 < self.config.minDegreesOfFreedom:
238  if name in self.config.allowFailure:
239  self.log.warn("Unable to measure aperture correction for '%s': "
240  "only %d sources, but require at least %d." %
241  (name, len(subset2), self.config.minDegreesOfFreedom+1))
242  continue
243  raise RuntimeError("Unable to measure aperture correction for required algorithm '%s': "
244  "only %d sources, but require at least %d." %
245  (name, len(subset2), self.config.minDegreesOfFreedom+1))
246 
247  # If we don't have enough data points to constrain the fit, reduce the order until we do
248  ctrl = self.config.fitConfig.makeControl()
249  while len(subset2) - ctrl.computeSize() < self.config.minDegreesOfFreedom:
250  if ctrl.orderX > 0:
251  ctrl.orderX -= 1
252  if ctrl.orderY > 0:
253  ctrl.orderY -= 1
254 
255  # Fill numpy arrays with positions and the ratio of the reference flux to the to-correct flux
256  x = numpy.zeros(len(subset2), dtype=float)
257  y = numpy.zeros(len(subset2), dtype=float)
258  apCorrData = numpy.zeros(len(subset2), dtype=float)
259  indices = numpy.arange(len(subset2), dtype=int)
260  for n, record in enumerate(subset2):
261  x[n] = record.getX()
262  y[n] = record.getY()
263  apCorrData[n] = record.get(self.refFluxKeys.flux)/record.get(keys.flux)
264 
265  for _i in range(self.config.numIter):
266 
267  # Do the fit, save it in the output map
268  apCorrField = ChebyshevBoundedField.fit(bbox, x, y, apCorrData, ctrl)
269 
270  if display:
271  plotApCorr(bbox, x, y, apCorrData, apCorrField, "%s, iteration %d" % (name, _i))
272 
273  # Compute errors empirically, using the RMS difference between the true reference flux and the
274  # corrected to-be-corrected flux.
275  apCorrDiffs = apCorrField.evaluate(x, y)
276  apCorrDiffs -= apCorrData
277  apCorrErr = numpy.mean(apCorrDiffs**2)**0.5
278 
279  # Clip bad data points
280  apCorrDiffLim = self.config.numSigmaClip * apCorrErr
281  with numpy.errstate(invalid="ignore"): # suppress NAN warning
282  keep = numpy.fabs(apCorrDiffs) <= apCorrDiffLim
283  x = x[keep]
284  y = y[keep]
285  apCorrData = apCorrData[keep]
286  indices = indices[keep]
287 
288  # Final fit after clipping
289  apCorrField = ChebyshevBoundedField.fit(bbox, x, y, apCorrData, ctrl)
290 
291  self.log.info("Aperture correction for %s: RMS %f from %d" %
292  (name, numpy.mean((apCorrField.evaluate(x, y) - apCorrData)**2)**0.5, len(indices)))
293 
294  if display:
295  plotApCorr(bbox, x, y, apCorrData, apCorrField, "%s, final" % (name,))
296 
297  # Save the result in the output map
298  # The error is constant spatially (we could imagine being
299  # more clever, but we're not yet sure if it's worth the effort).
300  # We save the errors as a 0th-order ChebyshevBoundedField
301  apCorrMap[fluxName] = apCorrField
302  apCorrErrCoefficients = numpy.array([[apCorrErr]], dtype=float)
303  apCorrMap[fluxSigmaName] = ChebyshevBoundedField(bbox, apCorrErrCoefficients)
304 
305  # Record which sources were used
306  for i in indices:
307  subset2[i].set(keys.used, True)
308 
309  return Struct(
310  apCorrMap=apCorrMap,
311  )
312 
313 
314 def plotApCorr(bbox, xx, yy, zzMeasure, field, title):
315  """Plot aperture correction fit residuals
316 
317  There are two subplots: residuals against x and y.
318 
319  Intended for debugging.
320 
321  @param bbox Bounding box (for bounds)
322  @param xx x coordinates
323  @param yy y coordinates
324  @param zzMeasure Measured value of the aperture correction
325  @param field Fit aperture correction field
326  @param title Title for plot
327  """
328  import matplotlib.pyplot as plt
329 
330  zzFit = field.evaluate(xx, yy)
331  residuals = zzMeasure - zzFit
332 
333  fig, axes = plt.subplots(2, 1)
334 
335  axes[0].scatter(xx, residuals, s=2, marker='o', lw=0, alpha=0.3)
336  axes[1].scatter(yy, residuals, s=2, marker='o', lw=0, alpha=0.3)
337  for ax in axes:
338  ax.set_ylabel("Residual")
339  ax.set_ylim(0.9*residuals.min(), 1.1*residuals.max())
340  axes[0].set_xlabel("x")
341  axes[0].set_xlim(bbox.getMinX(), bbox.getMaxX())
342  axes[1].set_xlabel("y")
343  axes[1].set_xlim(bbox.getMinY(), bbox.getMaxY())
344  plt.suptitle(title)
345 
346  plt.show()
def plotApCorr(bbox, xx, yy, zzMeasure, field, title)
def run(self, exposure, catalog)
Measure aperture correction.
def __init__(self, schema, kwds)
Construct a MeasureApCorrTask.