lsst.meas.algorithms  15.0-10-g113cadf7+1
measureApCorr.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 
24 __all__ = ("MeasureApCorrConfig", "MeasureApCorrTask")
25 
26 import numpy
27 
28 import lsst.pex.config
29 from lsst.afw.image import ApCorrMap
30 from lsst.afw.math import ChebyshevBoundedField, ChebyshevBoundedFieldConfig
31 from lsst.pipe.base import Task, Struct
32 from lsst.meas.base.apCorrRegistry import getApCorrNameSet
33 
34 from . import flaggedStarSelector # noqa register FlaggedStarSelectorTask
35 from .starSelector import starSelectorRegistry
36 
37 
38 class FluxKeys:
39  """A collection of keys for a given flux measurement algorithm
40  """
41  __slots__ = ("flux", "err", "flag", "used") # prevent accidentally adding fields
42 
43  def __init__(self, name, schema):
44  """Construct a FluxKeys
45 
46  @parma[in] name name of flux measurement algorithm, e.g. "base_PsfFlux"
47  @param[in,out] schema catalog schema containing the flux field
48  read: {name}_flux, {name}_fluxSigma, {name}_flag
49  added: apcorr_{name}_used
50  """
51  self.flux = schema.find(name + "_flux").key
52  self.err = schema.find(name + "_fluxSigma").key
53  self.flag = schema.find(name + "_flag").key
54  self.used = schema.addField("apcorr_" + name + "_used", type="Flag",
55  doc="set if source was used in measuring aperture correction")
56 
57 # The following block adds links to these tasks from the Task Documentation page.
58 
64 
65 
66 class MeasureApCorrConfig(lsst.pex.config.Config):
67  """!Configuration for MeasureApCorrTask
68  """
69  refFluxName = lsst.pex.config.Field(
70  doc="Field name prefix for the flux other measurements should be aperture corrected to match",
71  dtype=str,
72  default="slot_CalibFlux",
73  )
74  starSelector = starSelectorRegistry.makeField(
75  doc="Selector that sets the stars that aperture corrections will be measured from",
76  default="flagged",
77  )
78  minDegreesOfFreedom = lsst.pex.config.RangeField(
79  doc="Minimum number of degrees of freedom (# of valid data points - # of parameters);" +
80  " if this is exceeded, the order of the fit is decreased (in both dimensions), and" +
81  " if we can't decrease it enough, we'll raise ValueError.",
82  dtype=int,
83  default=1,
84  min=1,
85  )
86  fitConfig = lsst.pex.config.ConfigField(
87  doc="Configuration used in fitting the aperture correction fields",
88  dtype=ChebyshevBoundedFieldConfig,
89  )
90  numIter = lsst.pex.config.Field(
91  doc="Number of iterations for sigma clipping",
92  dtype=int,
93  default=4,
94  )
95  numSigmaClip = lsst.pex.config.Field(
96  doc="Number of standard devisations to clip at",
97  dtype=float,
98  default=3.0,
99  )
100  allowFailure = lsst.pex.config.ListField(
101  doc="Allow these measurement algorithms to fail without an exception",
102  dtype=str,
103  default=[],
104  )
105 
106  def validate(self):
107  lsst.pex.config.Config.validate(self)
108  if self.starSelector.target.usesMatches:
109  raise lsst.pex.config.FieldValidationError(
110  "Star selectors that require matches are not permitted"
111  )
112 
113 
114 class MeasureApCorrTask(Task):
115  """!Task to measure aperture correction
116 
117  @section measAlg_MeasureApCorrTask_Contents Contents
118 
119  - @ref measAlg_MeasureApCorrTask_Purpose
120  - @ref measAlg_MeasureApCorrTask_Config
121  - @ref measAlg_MeasureApCorrTask_Debug
122 
123  @section measAlg_MeasureApCorrTask_Purpose Description
124 
125  @copybrief MeasureApCorrTask
126 
127  This task measures aperture correction for the flux fields returned by
128  lsst.meas.base.getApCorrNameSet()
129 
130  The main method is @ref MeasureApCorrTask.run "run".
131 
132  @section measAlg_MeasureApCorrTask_Config Configuration parameters
133 
134  See @ref MeasureApCorrConfig
135 
136  @section measAlg_MeasureApCorrTask_Debug Debug variables
137 
138  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a flag
139  `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug for more about `debug.py`.
140 
141  MeasureApCorrTask has a debug dictionary containing a single boolean key:
142  <dl>
143  <dt>display
144  <dd>If True: will show plots as aperture corrections are fitted
145  </dl>
146 
147  For example, put something like:
148  @code{.py}
149  import lsstDebug
150  def DebugInfo(name):
151  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
152  if name == "lsst.meas.algorithms.measureApCorr":
153  di.display = dict(
154  unsubtracted = 1,
155  subtracted = 2,
156  background = 3,
157  )
158 
159  return di
160 
161  lsstDebug.Info = DebugInfo
162  @endcode
163  into your `debug.py` file and run your command-line task with the `--debug` flag (or `import debug`).
164  """
165  ConfigClass = MeasureApCorrConfig
166  _DefaultName = "measureApCorr"
167 
168  def __init__(self, schema, **kwds):
169  """!Construct a MeasureApCorrTask
170 
171  For every name in lsst.meas.base.getApCorrNameSet():
172  - If the corresponding flux fields exist in the schema:
173  - Add a new field apcorr_{name}_used
174  - Add an entry to the self.toCorrect dict
175  - Otherwise silently skip the name
176  """
177  Task.__init__(self, **kwds)
178  self.refFluxKeys = FluxKeys(self.config.refFluxName, schema)
179  self.toCorrect = {} # dict of flux field name prefix: FluxKeys instance
180  for name in getApCorrNameSet():
181  try:
182  self.toCorrect[name] = FluxKeys(name, schema)
183  except KeyError:
184  # if a field in the registry is missing, just ignore it.
185  pass
186  self.makeSubtask("starSelector", schema=schema)
187 
188  def run(self, exposure, catalog):
189  """!Measure aperture correction
190 
191  @param[in] exposure Exposure aperture corrections are being measured
192  on. Aside from the bounding box, the exposure
193  is only used by the starSelector subtask (which
194  may need it to construct PsfCandidates, as
195  PsfCanidate construction can do some filtering).
196  The output aperture correction map is *not*
197  added to the exposure; this is left to the
198  caller.
199 
200  @param[in] catalog SourceCatalog containing measurements to be used
201  to compute aperturecorrections.
202 
203  @return an lsst.pipe.base.Struct containing:
204  - apCorrMap: an aperture correction map (lsst.afw.image.ApCorrMap) that contains two entries
205  for each flux field:
206  - flux field (e.g. base_PsfFlux_flux): 2d model
207  - flux sigma field (e.g. base_PsfFlux_fluxSigma): 2d model of error
208  """
209  bbox = exposure.getBBox()
210  import lsstDebug
211  display = lsstDebug.Info(__name__).display
212 
213  self.log.info("Measuring aperture corrections for %d flux fields" % (len(self.toCorrect),))
214  # First, create a subset of the catalog that contains only selected stars
215  # with non-flagged reference fluxes.
216  subset1 = [record for record in self.starSelector.selectStars(exposure, catalog).starCat
217  if (not record.get(self.refFluxKeys.flag) and
218  numpy.isfinite(record.get(self.refFluxKeys.flux)))]
219 
220  apCorrMap = ApCorrMap()
221 
222  # Outer loop over the fields we want to correct
223  for name, keys in self.toCorrect.items():
224  fluxName = name + "_flux"
225  fluxSigmaName = name + "_fluxSigma"
226 
227  # Create a more restricted subset with only the objects where the to-be-correct flux
228  # is not flagged.
229  fluxes = numpy.fromiter((record.get(keys.flux) for record in subset1), float)
230  with numpy.errstate(invalid="ignore"): # suppress NAN warnings
231  isGood = numpy.logical_and.reduce([
232  numpy.fromiter((not record.get(keys.flag) for record in subset1), bool),
233  numpy.isfinite(fluxes),
234  fluxes > 0.0,
235  ])
236  subset2 = [record for record, good in zip(subset1, isGood) if good]
237 
238  # Check that we have enough data points that we have at least the minimum of degrees of
239  # freedom specified in the config.
240  if len(subset2) - 1 < self.config.minDegreesOfFreedom:
241  if name in self.config.allowFailure:
242  self.log.warn("Unable to measure aperture correction for '%s': "
243  "only %d sources, but require at least %d." %
244  (name, len(subset2), self.config.minDegreesOfFreedom+1))
245  continue
246  raise RuntimeError("Unable to measure aperture correction for required algorithm '%s': "
247  "only %d sources, but require at least %d." %
248  (name, len(subset2), self.config.minDegreesOfFreedom+1))
249 
250  # If we don't have enough data points to constrain the fit, reduce the order until we do
251  ctrl = self.config.fitConfig.makeControl()
252  while len(subset2) - ctrl.computeSize() < self.config.minDegreesOfFreedom:
253  if ctrl.orderX > 0:
254  ctrl.orderX -= 1
255  if ctrl.orderY > 0:
256  ctrl.orderY -= 1
257 
258  # Fill numpy arrays with positions and the ratio of the reference flux to the to-correct flux
259  x = numpy.zeros(len(subset2), dtype=float)
260  y = numpy.zeros(len(subset2), dtype=float)
261  apCorrData = numpy.zeros(len(subset2), dtype=float)
262  indices = numpy.arange(len(subset2), dtype=int)
263  for n, record in enumerate(subset2):
264  x[n] = record.getX()
265  y[n] = record.getY()
266  apCorrData[n] = record.get(self.refFluxKeys.flux)/record.get(keys.flux)
267 
268  for _i in range(self.config.numIter):
269 
270  # Do the fit, save it in the output map
271  apCorrField = ChebyshevBoundedField.fit(bbox, x, y, apCorrData, ctrl)
272 
273  if display:
274  plotApCorr(bbox, x, y, apCorrData, apCorrField, "%s, iteration %d" % (name, _i))
275 
276  # Compute errors empirically, using the RMS difference between the true reference flux and the
277  # corrected to-be-corrected flux.
278  apCorrDiffs = apCorrField.evaluate(x, y)
279  apCorrDiffs -= apCorrData
280  apCorrErr = numpy.mean(apCorrDiffs**2)**0.5
281 
282  # Clip bad data points
283  apCorrDiffLim = self.config.numSigmaClip * apCorrErr
284  with numpy.errstate(invalid="ignore"): # suppress NAN warning
285  keep = numpy.fabs(apCorrDiffs) <= apCorrDiffLim
286  x = x[keep]
287  y = y[keep]
288  apCorrData = apCorrData[keep]
289  indices = indices[keep]
290 
291  # Final fit after clipping
292  apCorrField = ChebyshevBoundedField.fit(bbox, x, y, apCorrData, ctrl)
293 
294  self.log.info("Aperture correction for %s: RMS %f from %d" %
295  (name, numpy.mean((apCorrField.evaluate(x, y) - apCorrData)**2)**0.5, len(indices)))
296 
297  if display:
298  plotApCorr(bbox, x, y, apCorrData, apCorrField, "%s, final" % (name,))
299 
300  # Save the result in the output map
301  # The error is constant spatially (we could imagine being
302  # more clever, but we're not yet sure if it's worth the effort).
303  # We save the errors as a 0th-order ChebyshevBoundedField
304  apCorrMap[fluxName] = apCorrField
305  apCorrErrCoefficients = numpy.array([[apCorrErr]], dtype=float)
306  apCorrMap[fluxSigmaName] = ChebyshevBoundedField(bbox, apCorrErrCoefficients)
307 
308  # Record which sources were used
309  for i in indices:
310  subset2[i].set(keys.used, True)
311 
312  return Struct(
313  apCorrMap=apCorrMap,
314  )
315 
316 
317 def plotApCorr(bbox, xx, yy, zzMeasure, field, title):
318  """Plot aperture correction fit residuals
319 
320  There are two subplots: residuals against x and y.
321 
322  Intended for debugging.
323 
324  @param bbox Bounding box (for bounds)
325  @param xx x coordinates
326  @param yy y coordinates
327  @param zzMeasure Measured value of the aperture correction
328  @param field Fit aperture correction field
329  @param title Title for plot
330  """
331  import matplotlib.pyplot as plt
332 
333  zzFit = field.evaluate(xx, yy)
334  residuals = zzMeasure - zzFit
335 
336  fig, axes = plt.subplots(2, 1)
337 
338  axes[0].scatter(xx, residuals, s=2, marker='o', lw=0, alpha=0.3)
339  axes[1].scatter(yy, residuals, s=2, marker='o', lw=0, alpha=0.3)
340  for ax in axes:
341  ax.set_ylabel("Residual")
342  ax.set_ylim(0.9*residuals.min(), 1.1*residuals.max())
343  axes[0].set_xlabel("x")
344  axes[0].set_xlim(bbox.getMinX(), bbox.getMaxX())
345  axes[1].set_xlabel("y")
346  axes[1].set_xlim(bbox.getMinY(), bbox.getMaxY())
347  plt.suptitle(title)
348 
349  plt.show()
def plotApCorr(bbox, xx, yy, zzMeasure, field, title)
def run(self, exposure, catalog)
Measure aperture correction.
def __init__(self, schema, kwds)
Construct a MeasureApCorrTask.