lsst.meas.algorithms  14.0-19-gbbfecc2c
measureApCorr.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 from __future__ import absolute_import, division
24 
25 __all__ = ("MeasureApCorrConfig", "MeasureApCorrTask")
26 
27 from builtins import zip
28 from builtins import range
29 from builtins import object
30 import numpy
31 
32 import lsst.pex.config
33 from lsst.afw.image import ApCorrMap
34 from lsst.afw.math import ChebyshevBoundedField, ChebyshevBoundedFieldConfig
35 from lsst.pipe.base import Task, Struct
36 from lsst.meas.base.apCorrRegistry import getApCorrNameSet
37 
38 from . import flaggedStarSelector # noqa register FlaggedStarSelectorTask
39 from .starSelector import starSelectorRegistry
40 
41 
42 class FluxKeys(object):
43  """A collection of keys for a given flux measurement algorithm
44  """
45  __slots__ = ("flux", "err", "flag", "used") # prevent accidentally adding fields
46 
47  def __init__(self, name, schema):
48  """Construct a FluxKeys
49 
50  @parma[in] name name of flux measurement algorithm, e.g. "base_PsfFlux"
51  @param[in,out] schema catalog schema containing the flux field
52  read: {name}_flux, {name}_fluxSigma, {name}_flag
53  added: apcorr_{name}_used
54  """
55  self.flux = schema.find(name + "_flux").key
56  self.err = schema.find(name + "_fluxSigma").key
57  self.flag = schema.find(name + "_flag").key
58  self.used = schema.addField("apcorr_" + name + "_used", type="Flag",
59  doc="set if source was used in measuring aperture correction")
60 
61 # The following block adds links to these tasks from the Task Documentation page.
62 
68 
69 
70 class MeasureApCorrConfig(lsst.pex.config.Config):
71  """!Configuration for MeasureApCorrTask
72  """
73  refFluxName = lsst.pex.config.Field(
74  doc="Field name prefix for the flux other measurements should be aperture corrected to match",
75  dtype=str,
76  default="slot_CalibFlux",
77  )
78  starSelector = starSelectorRegistry.makeField(
79  doc="Selector that sets the stars that aperture corrections will be measured from",
80  default="flagged",
81  )
82  minDegreesOfFreedom = lsst.pex.config.RangeField(
83  doc="Minimum number of degrees of freedom (# of valid data points - # of parameters);" +
84  " if this is exceeded, the order of the fit is decreased (in both dimensions), and" +
85  " if we can't decrease it enough, we'll raise ValueError.",
86  dtype=int,
87  default=1,
88  min=1,
89  )
90  fitConfig = lsst.pex.config.ConfigField(
91  doc="Configuration used in fitting the aperture correction fields",
92  dtype=ChebyshevBoundedFieldConfig,
93  )
94  numIter = lsst.pex.config.Field(
95  doc="Number of iterations for sigma clipping",
96  dtype=int,
97  default=4,
98  )
99  numSigmaClip = lsst.pex.config.Field(
100  doc="Number of standard devisations to clip at",
101  dtype=float,
102  default=3.0,
103  )
104  allowFailure = lsst.pex.config.ListField(
105  doc="Allow these measurement algorithms to fail without an exception",
106  dtype=str,
107  default=[],
108  )
109 
110  def validate(self):
111  lsst.pex.config.Config.validate(self)
112  if self.starSelector.target.usesMatches:
113  raise lsst.pex.config.FieldValidationError(
114  "Star selectors that require matches are not permitted"
115  )
116 
117 
118 class MeasureApCorrTask(Task):
119  """!Task to measure aperture correction
120 
121  \section measAlg_MeasureApCorrTask_Contents Contents
122 
123  - \ref measAlg_MeasureApCorrTask_Purpose
124  - \ref measAlg_MeasureApCorrTask_Config
125  - \ref measAlg_MeasureApCorrTask_Debug
126 
127  \section measAlg_MeasureApCorrTask_Purpose Description
128 
129  \copybrief MeasureApCorrTask
130 
131  This task measures aperture correction for the flux fields returned by
132  lsst.meas.base.getApCorrNameSet()
133 
134  The main method is \ref MeasureApCorrTask.run "run".
135 
136  \section measAlg_MeasureApCorrTask_Config Configuration parameters
137 
138  See \ref MeasureApCorrConfig
139 
140  \section measAlg_MeasureApCorrTask_Debug Debug variables
141 
142  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a flag
143  `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug for more about `debug.py`.
144 
145  MeasureApCorrTask has a debug dictionary containing a single boolean key:
146  <dl>
147  <dt>display
148  <dd>If True: will show plots as aperture corrections are fitted
149  </dl>
150 
151  For example, put something like:
152  @code{.py}
153  import lsstDebug
154  def DebugInfo(name):
155  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
156  if name == "lsst.meas.algorithms.measureApCorr":
157  di.display = dict(
158  unsubtracted = 1,
159  subtracted = 2,
160  background = 3,
161  )
162 
163  return di
164 
165  lsstDebug.Info = DebugInfo
166  @endcode
167  into your `debug.py` file and run your command-line task with the `--debug` flag (or `import debug`).
168  """
169  ConfigClass = MeasureApCorrConfig
170  _DefaultName = "measureApCorr"
171 
172  def __init__(self, schema, **kwds):
173  """!Construct a MeasureApCorrTask
174 
175  For every name in lsst.meas.base.getApCorrNameSet():
176  - If the corresponding flux fields exist in the schema:
177  - Add a new field apcorr_{name}_used
178  - Add an entry to the self.toCorrect dict
179  - Otherwise silently skip the name
180  """
181  Task.__init__(self, **kwds)
182  self.refFluxKeys = FluxKeys(self.config.refFluxName, schema)
183  self.toCorrect = {} # dict of flux field name prefix: FluxKeys instance
184  for name in getApCorrNameSet():
185  try:
186  self.toCorrect[name] = FluxKeys(name, schema)
187  except KeyError:
188  # if a field in the registry is missing, just ignore it.
189  pass
190  self.makeSubtask("starSelector", schema=schema)
191 
192  def run(self, exposure, catalog):
193  """!Measure aperture correction
194 
195  @param[in] exposure Exposure aperture corrections are being measured
196  on. Aside from the bounding box, the exposure
197  is only used by the starSelector subtask (which
198  may need it to construct PsfCandidates, as
199  PsfCanidate construction can do some filtering).
200  The output aperture correction map is *not*
201  added to the exposure; this is left to the
202  caller.
203 
204  @param[in] catalog SourceCatalog containing measurements to be used
205  to compute aperturecorrections.
206 
207  @return an lsst.pipe.base.Struct containing:
208  - apCorrMap: an aperture correction map (lsst.afw.image.ApCorrMap) that contains two entries
209  for each flux field:
210  - flux field (e.g. base_PsfFlux_flux): 2d model
211  - flux sigma field (e.g. base_PsfFlux_fluxSigma): 2d model of error
212  """
213  bbox = exposure.getBBox()
214  import lsstDebug
215  display = lsstDebug.Info(__name__).display
216 
217  self.log.info("Measuring aperture corrections for %d flux fields" % (len(self.toCorrect),))
218  # First, create a subset of the catalog that contains only selected stars
219  # with non-flagged reference fluxes.
220  subset1 = [record for record in self.starSelector.selectStars(exposure, catalog).starCat
221  if (not record.get(self.refFluxKeys.flag) and
222  numpy.isfinite(record.get(self.refFluxKeys.flux)))]
223 
224  apCorrMap = ApCorrMap()
225 
226  # Outer loop over the fields we want to correct
227  for name, keys in self.toCorrect.items():
228  fluxName = name + "_flux"
229  fluxSigmaName = name + "_fluxSigma"
230 
231  # Create a more restricted subset with only the objects where the to-be-correct flux
232  # is not flagged.
233  fluxes = numpy.fromiter((record.get(keys.flux) for record in subset1), float)
234  with numpy.errstate(invalid="ignore"): # suppress NAN warnings
235  isGood = numpy.logical_and.reduce([
236  numpy.fromiter((not record.get(keys.flag) for record in subset1), bool),
237  numpy.isfinite(fluxes),
238  fluxes > 0.0,
239  ])
240  subset2 = [record for record, good in zip(subset1, isGood) if good]
241 
242  # Check that we have enough data points that we have at least the minimum of degrees of
243  # freedom specified in the config.
244  if len(subset2) - 1 < self.config.minDegreesOfFreedom:
245  if name in self.config.allowFailure:
246  self.log.warn("Unable to measure aperture correction for '%s': "
247  "only %d sources, but require at least %d." %
248  (name, len(subset2), self.config.minDegreesOfFreedom+1))
249  continue
250  raise RuntimeError("Unable to measure aperture correction for required algorithm '%s': "
251  "only %d sources, but require at least %d." %
252  (name, len(subset2), self.config.minDegreesOfFreedom+1))
253 
254  # If we don't have enough data points to constrain the fit, reduce the order until we do
255  ctrl = self.config.fitConfig.makeControl()
256  while len(subset2) - ctrl.computeSize() < self.config.minDegreesOfFreedom:
257  if ctrl.orderX > 0:
258  ctrl.orderX -= 1
259  if ctrl.orderY > 0:
260  ctrl.orderY -= 1
261 
262  # Fill numpy arrays with positions and the ratio of the reference flux to the to-correct flux
263  x = numpy.zeros(len(subset2), dtype=float)
264  y = numpy.zeros(len(subset2), dtype=float)
265  apCorrData = numpy.zeros(len(subset2), dtype=float)
266  indices = numpy.arange(len(subset2), dtype=int)
267  for n, record in enumerate(subset2):
268  x[n] = record.getX()
269  y[n] = record.getY()
270  apCorrData[n] = record.get(self.refFluxKeys.flux)/record.get(keys.flux)
271 
272  for _i in range(self.config.numIter):
273 
274  # Do the fit, save it in the output map
275  apCorrField = ChebyshevBoundedField.fit(bbox, x, y, apCorrData, ctrl)
276 
277  if display:
278  plotApCorr(bbox, x, y, apCorrData, apCorrField, "%s, iteration %d" % (name, _i))
279 
280  # Compute errors empirically, using the RMS difference between the true reference flux and the
281  # corrected to-be-corrected flux.
282  apCorrDiffs = apCorrField.evaluate(x, y)
283  apCorrDiffs -= apCorrData
284  apCorrErr = numpy.mean(apCorrDiffs**2)**0.5
285 
286  # Clip bad data points
287  apCorrDiffLim = self.config.numSigmaClip * apCorrErr
288  with numpy.errstate(invalid="ignore"): # suppress NAN warning
289  keep = numpy.fabs(apCorrDiffs) <= apCorrDiffLim
290  x = x[keep]
291  y = y[keep]
292  apCorrData = apCorrData[keep]
293  indices = indices[keep]
294 
295  # Final fit after clipping
296  apCorrField = ChebyshevBoundedField.fit(bbox, x, y, apCorrData, ctrl)
297 
298  self.log.info("Aperture correction for %s: RMS %f from %d" %
299  (name, numpy.mean((apCorrField.evaluate(x, y) - apCorrData)**2)**0.5, len(indices)))
300 
301  if display:
302  plotApCorr(bbox, x, y, apCorrData, apCorrField, "%s, final" % (name,))
303 
304  # Save the result in the output map
305  # The error is constant spatially (we could imagine being
306  # more clever, but we're not yet sure if it's worth the effort).
307  # We save the errors as a 0th-order ChebyshevBoundedField
308  apCorrMap[fluxName] = apCorrField
309  apCorrErrCoefficients = numpy.array([[apCorrErr]], dtype=float)
310  apCorrMap[fluxSigmaName] = ChebyshevBoundedField(bbox, apCorrErrCoefficients)
311 
312  # Record which sources were used
313  for i in indices:
314  subset2[i].set(keys.used, True)
315 
316  return Struct(
317  apCorrMap=apCorrMap,
318  )
319 
320 
321 def plotApCorr(bbox, xx, yy, zzMeasure, field, title):
322  """Plot aperture correction fit residuals
323 
324  There are two subplots: residuals against x and y.
325 
326  Intended for debugging.
327 
328  @param bbox Bounding box (for bounds)
329  @param xx x coordinates
330  @param yy y coordinates
331  @param zzMeasure Measured value of the aperture correction
332  @param field Fit aperture correction field
333  @param title Title for plot
334  """
335  import matplotlib.pyplot as plt
336 
337  zzFit = field.evaluate(xx, yy)
338  residuals = zzMeasure - zzFit
339 
340  fig, axes = plt.subplots(2, 1)
341 
342  axes[0].scatter(xx, residuals, s=2, marker='o', lw=0, alpha=0.3)
343  axes[1].scatter(yy, residuals, s=2, marker='o', lw=0, alpha=0.3)
344  for ax in axes:
345  ax.set_ylabel("Residual")
346  ax.set_ylim(0.9*residuals.min(), 1.1*residuals.max())
347  axes[0].set_xlabel("x")
348  axes[0].set_xlim(bbox.getMinX(), bbox.getMaxX())
349  axes[1].set_xlabel("y")
350  axes[1].set_xlim(bbox.getMinY(), bbox.getMaxY())
351  plt.suptitle(title)
352 
353  plt.show()
def plotApCorr(bbox, xx, yy, zzMeasure, field, title)
def run(self, exposure, catalog)
Measure aperture correction.
def __init__(self, schema, kwds)
Construct a MeasureApCorrTask.