lsst.meas.algorithms  17.0.1-8-gd60f3cda+7
measureApCorr.py
Go to the documentation of this file.
1 #
2 # LSST Data Management System
3 #
4 # Copyright 2008-2017 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 
24 __all__ = ("MeasureApCorrConfig", "MeasureApCorrTask")
25 
26 import numpy
27 
28 import lsst.pex.config
29 from lsst.afw.image import ApCorrMap
30 from lsst.afw.math import ChebyshevBoundedField, ChebyshevBoundedFieldConfig
31 from lsst.pipe.base import Task, Struct
32 from lsst.meas.base.apCorrRegistry import getApCorrNameSet
33 
34 from .sourceSelector import sourceSelectorRegistry
35 
36 
37 class FluxKeys:
38  """A collection of keys for a given flux measurement algorithm
39  """
40  __slots__ = ("flux", "err", "flag", "used") # prevent accidentally adding fields
41 
42  def __init__(self, name, schema):
43  """Construct a FluxKeys
44 
45  @parma[in] name name of flux measurement algorithm, e.g. "base_PsfFlux"
46  @param[in,out] schema catalog schema containing the flux field
47  read: {name}_instFlux, {name}_instFluxErr, {name}_flag
48  added: apcorr_{name}_used
49  """
50  self.flux = schema.find(name + "_instFlux").key
51  self.err = schema.find(name + "_instFluxErr").key
52  self.flag = schema.find(name + "_flag").key
53  self.used = schema.addField("apcorr_" + name + "_used", type="Flag",
54  doc="set if source was used in measuring aperture correction")
55 
56 # The following block adds links to these tasks from the Task Documentation page.
57 
63 
64 
65 class MeasureApCorrConfig(lsst.pex.config.Config):
66  """!Configuration for MeasureApCorrTask
67  """
68  refFluxName = lsst.pex.config.Field(
69  doc="Field name prefix for the flux other measurements should be aperture corrected to match",
70  dtype=str,
71  default="slot_CalibFlux",
72  )
73  sourceSelector = sourceSelectorRegistry.makeField(
74  doc="Selector that sets the stars that aperture corrections will be measured from.",
75  default="flagged",
76  )
77  minDegreesOfFreedom = lsst.pex.config.RangeField(
78  doc="Minimum number of degrees of freedom (# of valid data points - # of parameters);"
79  " if this is exceeded, the order of the fit is decreased (in both dimensions), and"
80  " if we can't decrease it enough, we'll raise ValueError.",
81  dtype=int,
82  default=1,
83  min=1,
84  )
85  fitConfig = lsst.pex.config.ConfigField(
86  doc="Configuration used in fitting the aperture correction fields",
87  dtype=ChebyshevBoundedFieldConfig,
88  )
89  numIter = lsst.pex.config.Field(
90  doc="Number of iterations for sigma clipping",
91  dtype=int,
92  default=4,
93  )
94  numSigmaClip = lsst.pex.config.Field(
95  doc="Number of standard devisations to clip at",
96  dtype=float,
97  default=3.0,
98  )
99  allowFailure = lsst.pex.config.ListField(
100  doc="Allow these measurement algorithms to fail without an exception",
101  dtype=str,
102  default=[],
103  )
104 
105  def validate(self):
106  lsst.pex.config.Config.validate(self)
107  if self.sourceSelector.target.usesMatches:
108  raise lsst.pex.config.FieldValidationError(
109  "Star selectors that require matches are not permitted"
110  )
111 
112 
113 class MeasureApCorrTask(Task):
114  r"""!Task to measure aperture correction
115 
116  @section measAlg_MeasureApCorrTask_Contents Contents
117 
118  - @ref measAlg_MeasureApCorrTask_Purpose
119  - @ref measAlg_MeasureApCorrTask_Config
120  - @ref measAlg_MeasureApCorrTask_Debug
121 
122  @section measAlg_MeasureApCorrTask_Purpose Description
123 
124  @copybrief MeasureApCorrTask
125 
126  This task measures aperture correction for the flux fields returned by
127  lsst.meas.base.getApCorrNameSet()
128 
129  The main method is @ref MeasureApCorrTask.run "run".
130 
131  @section measAlg_MeasureApCorrTask_Config Configuration parameters
132 
133  See @ref MeasureApCorrConfig
134 
135  @section measAlg_MeasureApCorrTask_Debug Debug variables
136 
137  The @link lsst.pipe.base.cmdLineTask.CmdLineTask command line task@endlink interface supports a flag
138  `--debug` to import `debug.py` from your `$PYTHONPATH`; see @ref baseDebug for more about `debug.py`.
139 
140  MeasureApCorrTask has a debug dictionary containing a single boolean key:
141  <dl>
142  <dt>display
143  <dd>If True: will show plots as aperture corrections are fitted
144  </dl>
145 
146  For example, put something like:
147  @code{.py}
148  import lsstDebug
149  def DebugInfo(name):
150  di = lsstDebug.getInfo(name) # N.b. lsstDebug.Info(name) would call us recursively
151  if name == "lsst.meas.algorithms.measureApCorr":
152  di.display = dict(
153  unsubtracted = 1,
154  subtracted = 2,
155  background = 3,
156  )
157 
158  return di
159 
160  lsstDebug.Info = DebugInfo
161  @endcode
162  into your `debug.py` file and run your command-line task with the `--debug` flag (or `import debug`).
163  """
164  ConfigClass = MeasureApCorrConfig
165  _DefaultName = "measureApCorr"
166 
167  def __init__(self, schema, **kwds):
168  """!Construct a MeasureApCorrTask
169 
170  For every name in lsst.meas.base.getApCorrNameSet():
171  - If the corresponding flux fields exist in the schema:
172  - Add a new field apcorr_{name}_used
173  - Add an entry to the self.toCorrect dict
174  - Otherwise silently skip the name
175  """
176  Task.__init__(self, **kwds)
177  self.refFluxKeys = FluxKeys(self.config.refFluxName, schema)
178  self.toCorrect = {} # dict of flux field name prefix: FluxKeys instance
179  for name in getApCorrNameSet():
180  try:
181  self.toCorrect[name] = FluxKeys(name, schema)
182  except KeyError:
183  # if a field in the registry is missing, just ignore it.
184  pass
185  self.makeSubtask("sourceSelector")
186 
187  def run(self, exposure, catalog):
188  """!Measure aperture correction
189 
190  @param[in] exposure Exposure aperture corrections are being measured
191  on. The bounding box is retrieved from it, and
192  it is passed to the sourceSelector.
193  The output aperture correction map is *not*
194  added to the exposure; this is left to the
195  caller.
196 
197  @param[in] catalog SourceCatalog containing measurements to be used
198  to compute aperturecorrections.
199 
200  @return an lsst.pipe.base.Struct containing:
201  - apCorrMap: an aperture correction map (lsst.afw.image.ApCorrMap) that contains two entries
202  for each flux field:
203  - flux field (e.g. base_PsfFlux_instFlux): 2d model
204  - flux sigma field (e.g. base_PsfFlux_instFluxErr): 2d model of error
205  """
206  bbox = exposure.getBBox()
207  import lsstDebug
208  display = lsstDebug.Info(__name__).display
209  doPause = lsstDebug.Info(__name__).doPause
210 
211  self.log.info("Measuring aperture corrections for %d flux fields" % (len(self.toCorrect),))
212  # First, create a subset of the catalog that contains only selected stars
213  # with non-flagged reference fluxes.
214  subset1 = [record for record in self.sourceSelector.run(catalog, exposure=exposure).sourceCat
215  if (not record.get(self.refFluxKeys.flag) and
216  numpy.isfinite(record.get(self.refFluxKeys.flux)))]
217 
218  apCorrMap = ApCorrMap()
219 
220  # Outer loop over the fields we want to correct
221  for name, keys in self.toCorrect.items():
222  fluxName = name + "_instFlux"
223  fluxErrName = name + "_instFluxErr"
224 
225  # Create a more restricted subset with only the objects where the to-be-correct flux
226  # is not flagged.
227  fluxes = numpy.fromiter((record.get(keys.flux) for record in subset1), float)
228  with numpy.errstate(invalid="ignore"): # suppress NAN warnings
229  isGood = numpy.logical_and.reduce([
230  numpy.fromiter((not record.get(keys.flag) for record in subset1), bool),
231  numpy.isfinite(fluxes),
232  fluxes > 0.0,
233  ])
234  subset2 = [record for record, good in zip(subset1, isGood) if good]
235 
236  # Check that we have enough data points that we have at least the minimum of degrees of
237  # freedom specified in the config.
238  if len(subset2) - 1 < self.config.minDegreesOfFreedom:
239  if name in self.config.allowFailure:
240  self.log.warn("Unable to measure aperture correction for '%s': "
241  "only %d sources, but require at least %d." %
242  (name, len(subset2), self.config.minDegreesOfFreedom+1))
243  continue
244  raise RuntimeError("Unable to measure aperture correction for required algorithm '%s': "
245  "only %d sources, but require at least %d." %
246  (name, len(subset2), self.config.minDegreesOfFreedom+1))
247 
248  # If we don't have enough data points to constrain the fit, reduce the order until we do
249  ctrl = self.config.fitConfig.makeControl()
250  while len(subset2) - ctrl.computeSize() < self.config.minDegreesOfFreedom:
251  if ctrl.orderX > 0:
252  ctrl.orderX -= 1
253  if ctrl.orderY > 0:
254  ctrl.orderY -= 1
255 
256  # Fill numpy arrays with positions and the ratio of the reference flux to the to-correct flux
257  x = numpy.zeros(len(subset2), dtype=float)
258  y = numpy.zeros(len(subset2), dtype=float)
259  apCorrData = numpy.zeros(len(subset2), dtype=float)
260  indices = numpy.arange(len(subset2), dtype=int)
261  for n, record in enumerate(subset2):
262  x[n] = record.getX()
263  y[n] = record.getY()
264  apCorrData[n] = record.get(self.refFluxKeys.flux)/record.get(keys.flux)
265 
266  for _i in range(self.config.numIter):
267 
268  # Do the fit, save it in the output map
269  apCorrField = ChebyshevBoundedField.fit(bbox, x, y, apCorrData, ctrl)
270 
271  if display:
272  plotApCorr(bbox, x, y, apCorrData, apCorrField, "%s, iteration %d" % (name, _i), doPause)
273 
274  # Compute errors empirically, using the RMS difference between the true reference flux and the
275  # corrected to-be-corrected flux.
276  apCorrDiffs = apCorrField.evaluate(x, y)
277  apCorrDiffs -= apCorrData
278  apCorrErr = numpy.mean(apCorrDiffs**2)**0.5
279 
280  # Clip bad data points
281  apCorrDiffLim = self.config.numSigmaClip * apCorrErr
282  with numpy.errstate(invalid="ignore"): # suppress NAN warning
283  keep = numpy.fabs(apCorrDiffs) <= apCorrDiffLim
284  x = x[keep]
285  y = y[keep]
286  apCorrData = apCorrData[keep]
287  indices = indices[keep]
288 
289  # Final fit after clipping
290  apCorrField = ChebyshevBoundedField.fit(bbox, x, y, apCorrData, ctrl)
291 
292  self.log.info("Aperture correction for %s: RMS %f from %d" %
293  (name, numpy.mean((apCorrField.evaluate(x, y) - apCorrData)**2)**0.5, len(indices)))
294 
295  if display:
296  plotApCorr(bbox, x, y, apCorrData, apCorrField, "%s, final" % (name,), doPause)
297 
298  # Save the result in the output map
299  # The error is constant spatially (we could imagine being
300  # more clever, but we're not yet sure if it's worth the effort).
301  # We save the errors as a 0th-order ChebyshevBoundedField
302  apCorrMap[fluxName] = apCorrField
303  apCorrErrCoefficients = numpy.array([[apCorrErr]], dtype=float)
304  apCorrMap[fluxErrName] = ChebyshevBoundedField(bbox, apCorrErrCoefficients)
305 
306  # Record which sources were used
307  for i in indices:
308  subset2[i].set(keys.used, True)
309 
310  return Struct(
311  apCorrMap=apCorrMap,
312  )
313 
314 
315 def plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause):
316  """Plot aperture correction fit residuals
317 
318  There are two subplots: residuals against x and y.
319 
320  Intended for debugging.
321 
322  @param bbox Bounding box (for bounds)
323  @param xx x coordinates
324  @param yy y coordinates
325  @param zzMeasure Measured value of the aperture correction
326  @param field Fit aperture correction field
327  @param title Title for plot
328  @param doPause Pause to inspect the residuals plot? If False,
329  there will be a 4 second delay to allow for
330  inspection of the plot before closing it and
331  moving on.
332  """
333  import matplotlib.pyplot as plt
334 
335  zzFit = field.evaluate(xx, yy)
336  residuals = zzMeasure - zzFit
337 
338  fig, axes = plt.subplots(2, 1)
339 
340  axes[0].scatter(xx, residuals, s=3, marker='o', lw=0, alpha=0.7)
341  axes[1].scatter(yy, residuals, s=3, marker='o', lw=0, alpha=0.7)
342  for ax in axes:
343  ax.set_ylabel("ApCorr Fit Residual")
344  ax.set_ylim(0.9*residuals.min(), 1.1*residuals.max())
345  axes[0].set_xlabel("x")
346  axes[0].set_xlim(bbox.getMinX(), bbox.getMaxX())
347  axes[1].set_xlabel("y")
348  axes[1].set_xlim(bbox.getMinY(), bbox.getMaxY())
349  plt.suptitle(title)
350 
351  if not doPause:
352  try:
353  plt.pause(4)
354  plt.close()
355  except Exception:
356  print("%s: plt.pause() failed. Please close plots when done." % __name__)
357  plt.show()
358  else:
359  print("%s: Please close plots when done." % __name__)
360  plt.show()
def run(self, exposure, catalog)
Measure aperture correction.
def __init__(self, schema, kwds)
Construct a MeasureApCorrTask.
def plotApCorr(bbox, xx, yy, zzMeasure, field, title, doPause)