lsst.meas.algorithms  14.0-9-g82279ae0+4
measureApCorr.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 from __future__ import absolute_import, division
24 
25 __all__ = ("MeasureApCorrConfig", "MeasureApCorrTask")
26 
27 from builtins import zip
28 from builtins import range
29 from builtins import object
30 import numpy
31 
32 import lsst.pex.config
33 from lsst.afw.image import ApCorrMap
34 from lsst.afw.math import ChebyshevBoundedField, ChebyshevBoundedFieldConfig
35 from lsst.pipe.base import Task, Struct
36 from lsst.meas.base.apCorrRegistry import getApCorrNameSet
37 
38 from . import flaggedStarSelector # noqa register FlaggedStarSelectorTask
39 from .starSelector import starSelectorRegistry
40 
41 
42 class FluxKeys(object):
43  """A collection of keys for a given flux measurement algorithm
44  """
45  __slots__ = ("flux", "err", "flag", "used") # prevent accidentally adding fields
46 
47  def __init__(self, name, schema):
48  """Construct a FluxKeys
49 
50  @parma[in] name name of flux measurement algorithm, e.g. "base_PsfFlux"
51  @param[in,out] schema catalog schema containing the flux field
52  read: {name}_flux, {name}_fluxSigma, {name}_flag
53  added: apcorr_{name}_used
54  """
55  self.flux = schema.find(name + "_flux").key
56  self.err = schema.find(name + "_fluxSigma").key
57  self.flag = schema.find(name + "_flag").key
58  self.used = schema.addField("apcorr_" + name + "_used", type="Flag",
59  doc="set if source was used in measuring aperture correction")
60 
61 # The following block adds links to these tasks from the Task Documentation page.
62 
68 
69 
70 class MeasureApCorrConfig(lsst.pex.config.Config):
71  """!Configuration for MeasureApCorrTask
72  """
73  refFluxName = lsst.pex.config.Field(
74  doc="Field name prefix for the flux other measurements should be aperture corrected to match",
75  dtype=str,
76  default="slot_CalibFlux",
77  )
78  starSelector = starSelectorRegistry.makeField(
79  doc="Selector that sets the stars that aperture corrections will be measured from",
80  default="flagged",
81  )
82  minDegreesOfFreedom = lsst.pex.config.RangeField(
83  doc="Minimum number of degrees of freedom (# of valid data points - # of parameters);" +
84  " if this is exceeded, the order of the fit is decreased (in both dimensions), and" +
85  " if we can't decrease it enough, we'll raise ValueError.",
86  dtype=int,
87  default=1,
88  min=1,
89  )
90  fitConfig = lsst.pex.config.ConfigField(
91  doc="Configuration used in fitting the aperture correction fields",
92  dtype=ChebyshevBoundedFieldConfig,
93  )
94  numIter = lsst.pex.config.Field(
95  doc="Number of iterations for sigma clipping",
96  dtype=int,
97  default=4,
98  )
99  numSigmaClip = lsst.pex.config.Field(
100  doc="Number of standard devisations to clip at",
101  dtype=float,
102  default=3.0,
103  )
104  allowFailure = lsst.pex.config.ListField(
105  doc="Allow these measurement algorithms to fail without an exception",
106  dtype=str,
107  default=[],
108  )
109 
110  def validate(self):
111  lsst.pex.config.Config.validate(self)
112  if self.starSelector.target.usesMatches:
113  raise lsst.pex.config.FieldValidationError(
114  "Star selectors that require matches are not permitted"
115  )
116 
117 
118 class MeasureApCorrTask(Task):
119  """!Task to measure aperture correction
120 
121  \section measAlg_MeasureApCorrTask_Contents Contents
122 
123  - \ref measAlg_MeasureApCorrTask_Purpose
124  - \ref measAlg_MeasureApCorrTask_Config
125  - \ref measAlg_MeasureApCorrTask_Debug
126 
127  \section measAlg_MeasureApCorrTask_Purpose Description
128 
129  \copybrief MeasureApCorrTask
130 
131  This task measures aperture correction for the flux fields returned by
132  lsst.meas.base.getApCorrNameSet()
133 
134  The main method is \ref MeasureApCorrTask.run "run".
135 
136  \section measAlg_MeasureApCorrTask_Config Configuration parameters
137 
138  See \ref MeasureApCorrConfig
139 
140  \section measAlg_MeasureApCorrTask_Debug Debug variables
141 
142  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a flag
143  `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug for more about `debug.py`.
144 
145  MeasureApCorrTask has a debug dictionary containing a single boolean key:
146  <dl>
147  <dt>display
148  <dd>If True: will show plots as aperture corrections are fitted
149  </dl>
150 
151  For example, put something like:
152  @code{.py}
153  import lsstDebug
154  def DebugInfo(name):
155  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
156  if name == "lsst.meas.algorithms.measureApCorr":
157  di.display = dict(
158  unsubtracted = 1,
159  subtracted = 2,
160  background = 3,
161  )
162 
163  return di
164 
165  lsstDebug.Info = DebugInfo
166  @endcode
167  into your `debug.py` file and run your command-line task with the `--debug` flag (or `import debug`).
168  """
169  ConfigClass = MeasureApCorrConfig
170  _DefaultName = "measureApCorr"
171 
172  def __init__(self, schema, **kwds):
173  """!Construct a MeasureApCorrTask
174 
175  For every name in lsst.meas.base.getApCorrNameSet():
176  - If the corresponding flux fields exist in the schema:
177  - Add a new field apcorr_{name}_used
178  - Add an entry to the self.toCorrect dict
179  - Otherwise silently skip the name
180  """
181  Task.__init__(self, **kwds)
182  self.refFluxKeys = FluxKeys(self.config.refFluxName, schema)
183  self.toCorrect = {} # dict of flux field name prefix: FluxKeys instance
184  for name in getApCorrNameSet():
185  try:
186  self.toCorrect[name] = FluxKeys(name, schema)
187  except KeyError:
188  # if a field in the registry is missing, just ignore it.
189  pass
190  self.makeSubtask("starSelector", schema=schema)
191 
192  def run(self, exposure, catalog):
193  """!Measure aperture correction
194 
195  @param[in] exposure Exposure aperture corrections are being measured
196  on. Aside from the bounding box, the exposure
197  is only used by the starSelector subtask (which
198  may need it to construct PsfCandidates, as
199  PsfCanidate construction can do some filtering).
200  The output aperture correction map is *not*
201  added to the exposure; this is left to the
202  caller.
203 
204  @param[in] catalog SourceCatalog containing measurements to be used
205  to compute aperturecorrections.
206 
207  @return an lsst.pipe.base.Struct containing:
208  - apCorrMap: an aperture correction map (lsst.afw.image.ApCorrMap) that contains two entries
209  for each flux field:
210  - flux field (e.g. base_PsfFlux_flux): 2d model
211  - flux sigma field (e.g. base_PsfFlux_fluxSigma): 2d model of error
212  """
213  bbox = exposure.getBBox()
214  import lsstDebug
215  display = lsstDebug.Info(__name__).display
216 
217  self.log.info("Measuring aperture corrections for %d flux fields" % (len(self.toCorrect),))
218  # First, create a subset of the catalog that contains only selected stars
219  # with non-flagged reference fluxes.
220  subset1 = [record for record in self.starSelector.selectStars(exposure, catalog).starCat
221  if (not record.get(self.refFluxKeys.flag) and
222  numpy.isfinite(record.get(self.refFluxKeys.flux)))]
223 
224  apCorrMap = ApCorrMap()
225 
226  # Outer loop over the fields we want to correct
227  for name, keys in self.toCorrect.items():
228  fluxName = name + "_flux"
229  fluxSigmaName = name + "_fluxSigma"
230 
231  # Create a more restricted subset with only the objects where the to-be-correct flux
232  # is not flagged.
233  fluxes = numpy.fromiter((record.get(keys.flux) for record in subset1), float)
234  isGood = numpy.logical_and.reduce([
235  numpy.fromiter((not record.get(keys.flag) for record in subset1), bool),
236  numpy.isfinite(fluxes),
237  fluxes > 0.0,
238  ])
239  subset2 = [record for record, good in zip(subset1, isGood) if good]
240 
241  # Check that we have enough data points that we have at least the minimum of degrees of
242  # freedom specified in the config.
243  if len(subset2) - 1 < self.config.minDegreesOfFreedom:
244  if name in self.config.allowFailure:
245  self.log.warn("Unable to measure aperture correction for '%s': "
246  "only %d sources, but require at least %d." %
247  (name, len(subset2), self.config.minDegreesOfFreedom+1))
248  continue
249  raise RuntimeError("Unable to measure aperture correction for required algorithm '%s': "
250  "only %d sources, but require at least %d." %
251  (name, len(subset2), self.config.minDegreesOfFreedom+1))
252 
253  # If we don't have enough data points to constrain the fit, reduce the order until we do
254  ctrl = self.config.fitConfig.makeControl()
255  while len(subset2) - ctrl.computeSize() < self.config.minDegreesOfFreedom:
256  if ctrl.orderX > 0:
257  ctrl.orderX -= 1
258  if ctrl.orderY > 0:
259  ctrl.orderY -= 1
260 
261  # Fill numpy arrays with positions and the ratio of the reference flux to the to-correct flux
262  x = numpy.zeros(len(subset2), dtype=float)
263  y = numpy.zeros(len(subset2), dtype=float)
264  apCorrData = numpy.zeros(len(subset2), dtype=float)
265  indices = numpy.arange(len(subset2), dtype=int)
266  for n, record in enumerate(subset2):
267  x[n] = record.getX()
268  y[n] = record.getY()
269  apCorrData[n] = record.get(self.refFluxKeys.flux)/record.get(keys.flux)
270 
271  for _i in range(self.config.numIter):
272 
273  # Do the fit, save it in the output map
274  apCorrField = ChebyshevBoundedField.fit(bbox, x, y, apCorrData, ctrl)
275 
276  if display:
277  plotApCorr(bbox, x, y, apCorrData, apCorrField, "%s, iteration %d" % (name, _i))
278 
279  # Compute errors empirically, using the RMS difference between the true reference flux and the
280  # corrected to-be-corrected flux.
281  apCorrDiffs = apCorrField.evaluate(x, y)
282  apCorrDiffs -= apCorrData
283  apCorrErr = numpy.mean(apCorrDiffs**2)**0.5
284 
285  # Clip bad data points
286  apCorrDiffLim = self.config.numSigmaClip * apCorrErr
287  keep = numpy.fabs(apCorrDiffs) <= apCorrDiffLim
288  x = x[keep]
289  y = y[keep]
290  apCorrData = apCorrData[keep]
291  indices = indices[keep]
292 
293  # Final fit after clipping
294  apCorrField = ChebyshevBoundedField.fit(bbox, x, y, apCorrData, ctrl)
295 
296  self.log.info("Aperture correction for %s: RMS %f from %d" %
297  (name, numpy.mean((apCorrField.evaluate(x, y) - apCorrData)**2)**0.5, len(indices)))
298 
299  if display:
300  plotApCorr(bbox, x, y, apCorrData, apCorrField, "%s, final" % (name,))
301 
302  # Save the result in the output map
303  # The error is constant spatially (we could imagine being
304  # more clever, but we're not yet sure if it's worth the effort).
305  # We save the errors as a 0th-order ChebyshevBoundedField
306  apCorrMap[fluxName] = apCorrField
307  apCorrErrCoefficients = numpy.array([[apCorrErr]], dtype=float)
308  apCorrMap[fluxSigmaName] = ChebyshevBoundedField(bbox, apCorrErrCoefficients)
309 
310  # Record which sources were used
311  for i in indices:
312  subset2[i].set(keys.used, True)
313 
314  return Struct(
315  apCorrMap=apCorrMap,
316  )
317 
318 
319 def plotApCorr(bbox, xx, yy, zzMeasure, field, title):
320  """Plot aperture correction fit residuals
321 
322  There are two subplots: residuals against x and y.
323 
324  Intended for debugging.
325 
326  @param bbox Bounding box (for bounds)
327  @param xx x coordinates
328  @param yy y coordinates
329  @param zzMeasure Measured value of the aperture correction
330  @param field Fit aperture correction field
331  @param title Title for plot
332  """
333  import matplotlib.pyplot as plt
334 
335  zzFit = field.evaluate(xx, yy)
336  residuals = zzMeasure - zzFit
337 
338  fig, axes = plt.subplots(2, 1)
339 
340  axes[0].scatter(xx, residuals, s=2, marker='o', lw=0, alpha=0.3)
341  axes[1].scatter(yy, residuals, s=2, marker='o', lw=0, alpha=0.3)
342  for ax in axes:
343  ax.set_ylabel("Residual")
344  ax.set_ylim(0.9*residuals.min(), 1.1*residuals.max())
345  axes[0].set_xlabel("x")
346  axes[0].set_xlim(bbox.getMinX(), bbox.getMaxX())
347  axes[1].set_xlabel("y")
348  axes[1].set_xlim(bbox.getMinY(), bbox.getMaxY())
349  plt.suptitle(title)
350 
351  plt.show()
def plotApCorr(bbox, xx, yy, zzMeasure, field, title)
def run(self, exposure, catalog)
Measure aperture correction.
def __init__(self, schema, kwds)
Construct a MeasureApCorrTask.