lsst.meas.base  16.0-6-g4a515f9+3
baseMeasurement.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2016 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 """Base measurement task, which subclassed by the single frame and forced measurement tasks.
24 """
25 import lsst.pipe.base
26 import lsst.pex.config
27 
28 from .pluginRegistry import PluginMap
29 from .exceptions import FatalAlgorithmError, MeasurementError
30 from .pluginsBase import BasePluginConfig, BasePlugin
31 from .noiseReplacer import NoiseReplacerConfig
32 
33 __all__ = ("BaseMeasurementPluginConfig", "BaseMeasurementPlugin",
34  "BaseMeasurementConfig", "BaseMeasurementTask")
35 
36 # Exceptions that the measurement tasks should always propagate up to their callers
37 FATAL_EXCEPTIONS = (MemoryError, FatalAlgorithmError)
38 
39 
41  """!
42  Base config class for all measurement plugins
43 
44  Most derived classes will want to override setDefaults() in order to customize
45  the default exceutionOrder.
46 
47  A derived class whose corresponding Plugin class implements measureN() should
48  additionally add a bool doMeasureN field to replace the bool class attribute
49  defined here.
50  """
51 
52  doMeasure = lsst.pex.config.Field(dtype=bool, default=True,
53  doc="whether to run this plugin in single-object mode")
54 
55  doMeasureN = False # replace this class attribute with a Field if measureN-capable
56 
57 
59  '''
60  Base class for all measurement plugins
61 
62  This is class is a placeholder for future behavior which will be shared only between
63  measurement plugins and is implemented for symmetry with the measurement base plugin
64  configuration class
65  '''
66  pass
67 
68 
69 class SourceSlotConfig(lsst.pex.config.Config):
70  """!
71  Slot configuration which assigns a particular named plugin to each of a set of
72  slots. Each slot allows a type of measurement to be fetched from the SourceTable
73  without knowing which algorithm was used to produced the data.
74 
75  NOTE: the default algorithm for each slot must be registered, even if the default is not used.
76  """
77 
78  centroid = lsst.pex.config.Field(dtype=str, default="base_SdssCentroid", optional=True,
79  doc="the name of the centroiding algorithm used to set source x,y")
80  shape = lsst.pex.config.Field(dtype=str, default="base_SdssShape", optional=True,
81  doc="the name of the algorithm used to set source moments parameters")
82  psfShape = lsst.pex.config.Field(dtype=str, default="base_SdssShape_psf", optional=True,
83  doc="the name of the algorithm used to set PSF moments parameters")
84  apFlux = lsst.pex.config.Field(dtype=str, default="base_CircularApertureFlux_12_0", optional=True,
85  doc="the name of the algorithm used to set the source aperture flux slot")
86  modelFlux = lsst.pex.config.Field(dtype=str, default="base_GaussianFlux", optional=True,
87  doc="the name of the algorithm used to set the source model flux slot")
88  psfFlux = lsst.pex.config.Field(dtype=str, default="base_PsfFlux", optional=True,
89  doc="the name of the algorithm used to set the source psf flux slot")
90  instFlux = lsst.pex.config.Field(dtype=str, default="base_GaussianFlux", optional=True,
91  doc="the name of the algorithm used to set the source inst flux slot")
92  calibFlux = lsst.pex.config.Field(dtype=str, default="base_CircularApertureFlux_12_0", optional=True,
93  doc="the name of the flux measurement algorithm used for calibration")
94 
95  def setupSchema(self, schema):
96  """Convenience method to setup a Schema's slots according to the config definition.
97 
98  This is defined in the Config class to support use in unit tests without needing
99  to construct a Task object.
100  """
101  aliases = schema.getAliasMap()
102  if self.centroid is not None:
103  aliases.set("slot_Centroid", self.centroid)
104  if self.shape is not None:
105  aliases.set("slot_Shape", self.shape)
106  if self.psfShape is not None:
107  aliases.set("slot_PsfShape", self.psfShape)
108  if self.apFlux is not None:
109  aliases.set("slot_ApFlux", self.apFlux)
110  if self.modelFlux is not None:
111  aliases.set("slot_ModelFlux", self.modelFlux)
112  if self.psfFlux is not None:
113  aliases.set("slot_PsfFlux", self.psfFlux)
114  if self.instFlux is not None:
115  aliases.set("slot_InstFlux", self.instFlux)
116  if self.calibFlux is not None:
117  aliases.set("slot_CalibFlux", self.calibFlux)
118 
119 
120 class BaseMeasurementConfig(lsst.pex.config.Config):
121  """!
122  Base config class for all measurement driver tasks.
123 
124  Subclasses should define the 'plugins' and 'undeblended' registries, e.g.:
125 
126  plugins = PluginBaseClass.registry.makeField(
127  multi=True,
128  default=[],
129  doc="Plugins to be run and their configuration"
130  )
131  undeblended = PluginBaseClass.registry.makeField(
132  multi=True,
133  default=[],
134  doc="Plugins to run on undeblended image"
135  )
136 
137  where PluginBaseClass is the appropriate base class of the plugin
138  (e.g., SingleFramePlugin or ForcedPlugin).
139  """
140 
141  slots = lsst.pex.config.ConfigField(
142  dtype=SourceSlotConfig,
143  doc="Mapping from algorithms to special aliases in Source."
144  )
145 
146  doReplaceWithNoise = lsst.pex.config.Field(
147  dtype=bool, default=True, optional=False,
148  doc='When measuring, replace other detected footprints with noise?')
149 
150  noiseReplacer = lsst.pex.config.ConfigField(
151  dtype=NoiseReplacerConfig,
152  doc="configuration that sets how to replace neighboring sources with noise"
153  )
154  undeblendedPrefix = lsst.pex.config.Field(
155  dtype=str, default="undeblended_",
156  doc="Prefix to give undeblended plugins"
157  )
158 
159  def validate(self):
160  lsst.pex.config.Config.validate(self)
161  if self.slots.centroid is not None and self.slots.centroid not in self.plugins.names:
162  raise ValueError("source centroid slot algorithm is not being run.")
163  if self.slots.shape is not None and self.slots.shape not in self.plugins.names:
164  raise ValueError("source shape slot algorithm '%s' is not being run." % self.slots.shape)
165  for slot in (self.slots.psfFlux, self.slots.apFlux, self.slots.modelFlux,
166  self.slots.instFlux, self.slots.calibFlux):
167  if slot is not None:
168  for name in self.plugins.names:
169  if len(name) <= len(slot) and name == slot[:len(name)]:
170  break
171  else:
172  raise ValueError("source flux slot algorithm '%s' is not being run." % slot)
173 
174 
179 
180 
181 class BaseMeasurementTask(lsst.pipe.base.Task):
182  """!
183  Ultimate base class for all measurement tasks.
184 
185  This base class for SingleFrameMeasurementTask and ForcedMeasurementTask mostly exists to share
186  code between the two, and generally should not be used directly.
187  """
188 
189  ConfigClass = BaseMeasurementConfig
190  _DefaultName = "measurement"
191 
192  def __init__(self, algMetadata=None, **kwds):
193  """!
194  Constructor; only called by derived classes.
195 
196  @param[in] algMetadata An lsst.daf.base.PropertyList that will be filled with metadata
197  about the plugins being run. If None, an empty PropertyList will
198  be created.
199  @param[in] **kwds Additional arguments passed to lsst.pipe.base.Task.__init__.
200 
201  This attaches two public attributes to the class for use by derived classes and parent tasks:
202  - plugins: an empty PluginMap, which will eventually contain all active plugins that will by
203  invoked by the run() method (to be filled by subclasses). This should be considered read-only.
204  - algMetadata: a lsst.daf.base.PropertyList that will contain additional information about the
205  active plugins to be saved with the output catalog (to be filled by subclasses).
206  """
207  super(BaseMeasurementTask, self).__init__(**kwds)
210  if algMetadata is None:
211  algMetadata = lsst.daf.base.PropertyList()
212  self.algMetadata = algMetadata
213 
214  def getPluginLogName(self, pluginName):
215  return self.log.getName() + '.' + pluginName
216 
217  def initializePlugins(self, **kwds):
218  """Initialize the plugins (and slots) according to the configuration.
219 
220  Derived class constructors should call this method to fill the self.plugins
221  attribute and add correspond output fields and slot aliases to the output schema.
222 
223  In addition to the attributes added by BaseMeasurementTask.__init__, a self.schema
224  attribute holding the output schema must also be present before this method is called, .
225 
226  Keyword arguments are forwarded directly to plugin constructors, allowing derived
227  classes to use plugins with different signatures.
228  """
229  # Make a place at the beginning for the centroid plugin to run first (because it's an OrderedDict,
230  # adding an empty element in advance means it will get run first when it's reassigned to the
231  # actual Plugin).
232  if self.config.slots.centroid is not None:
233  self.plugins[self.config.slots.centroid] = None
234  # Init the plugins, sorted by execution order. At the same time add to the schema
235  for executionOrder, name, config, PluginClass in sorted(self.config.plugins.apply()):
236  # Pass logName to the plugin if the plugin is marked as using it
237  # The task will use this name to log plugin errors, regardless.
238  if hasattr(PluginClass, "hasLogName") and PluginClass.hasLogName:
239  self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata,
240  logName=self.getPluginLogName(name), **kwds)
241  else:
242  self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata, **kwds)
243 
244  # In rare circumstances (usually tests), the centroid slot not be coming from an algorithm,
245  # which means we'll have added something we don't want to the plugins map, and we should
246  # remove it.
247  if self.config.slots.centroid is not None and self.plugins[self.config.slots.centroid] is None:
248  del self.plugins[self.config.slots.centroid]
249  # Initialize the plugins to run on the undeblended image
250  for executionOrder, name, config, PluginClass in sorted(self.config.undeblended.apply()):
251  undeblendedName = self.config.undeblendedPrefix + name
252  self.undeblendedPlugins[name] = PluginClass(config, undeblendedName, metadata=self.algMetadata,
253  **kwds)
254 
255  def callMeasure(self, measRecord, *args, **kwds):
256  """!
257  Call the measure() method on all plugins, handling exceptions in a consistent way.
258 
259  @param[in,out] measRecord lsst.afw.table.SourceRecord that corresponds to the object being
260  measured, and where outputs should be written.
261  @param[in] *args Positional arguments forwarded to Plugin.measure()
262  @param[in] **kwds Keyword arguments. Two are handled locally:
263  - beginOrder: beginning execution order (inclusive): measurements with
264  executionOrder < beginOrder are not executed. None for no limit.
265  - endOrder: ending execution order (exclusive): measurements with
266  executionOrder >= endOrder are not executed. None for no limit.
267  the rest are forwarded to Plugin.measure()
268 
269  This method can be used with plugins that have different signatures; the only requirement is that
270  'measRecord' be the first argument. Subsequent positional arguments and keyword arguments are
271  forwarded directly to the plugin.
272 
273  This method should be considered "protected"; it is intended for use by derived classes, not users.
274  """
275  beginOrder = kwds.pop("beginOrder", None)
276  endOrder = kwds.pop("endOrder", None)
277  for plugin in self.plugins.iter():
278  if beginOrder is not None and plugin.getExecutionOrder() < beginOrder:
279  continue
280  if endOrder is not None and plugin.getExecutionOrder() >= endOrder:
281  break
282  self.doMeasurement(plugin, measRecord, *args, **kwds)
283 
284  def doMeasurement(self, plugin, measRecord, *args, **kwds):
285  """!
286  Call the measure() method on the nominated plugin, handling exceptions in a consistent way.
287 
288  @param[in] plugin Plugin that will measure
289  @param[in,out] measRecord lsst.afw.table.SourceRecord that corresponds to the object being
290  measured, and where outputs should be written.
291  @param[in] *args Positional arguments forwarded to plugin.measure()
292  @param[in] **kwds Keyword arguments forwarded to plugin.measure()
293 
294  This method can be used with plugins that have different signatures; the only requirement is that
295  the 'plugin' and 'measRecord' be the first two arguments. Subsequent positional arguments and
296  keyword arguments are forwarded directly to the plugin.
297 
298  This method should be considered "protected"; it is intended for use by derived classes, not users.
299  """
300  try:
301  plugin.measure(measRecord, *args, **kwds)
302  except FATAL_EXCEPTIONS:
303  raise
304  except MeasurementError as error:
305  lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).debug(
306  "MeasurementError in %s.measure on record %s: %s"
307  % (plugin.name, measRecord.getId(), error))
308  plugin.fail(measRecord, error)
309  except Exception as error:
310  lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).debug(
311  "Exception in %s.measure on record %s: %s"
312  % (plugin.name, measRecord.getId(), error))
313  plugin.fail(measRecord)
314 
315  def callMeasureN(self, measCat, *args, **kwds):
316  """!
317  Call the measureN() method on all plugins, handling exceptions in a consistent way.
318 
319  @param[in,out] measCat lsst.afw.table.SourceCatalog containing records for just
320  the source family to be measured, and where outputs should
321  be written.
322  @param[in] beginOrder beginning execution order (inclusive): measurements with
323  executionOrder < beginOrder are not executed. None for no limit.
324  @param[in] endOrder ending execution order (exclusive): measurements with
325  executionOrder >= endOrder are not executed. None for no limit.
326  @param[in] *args Positional arguments forwarded to Plugin.measure()
327  @param[in] **kwds Keyword arguments. Two are handled locally:
328  - beginOrder: beginning execution order (inclusive): measurements with
329  executionOrder < beginOrder are not executed. None for no limit.
330  - endOrder: ending execution order (exclusive): measurements with
331  executionOrder >= endOrder are not executed. None for no limit.
332  the rest are forwarded to Plugin.measure()
333 
334  This method can be used with plugins that have different signatures; the only requirement is that
335  'measRecord' be the first argument. Subsequent positional arguments and keyword arguments are
336  forwarded directly to the plugin.
337 
338  This method should be considered "protected"; it is intended for use by derived classes, not users.
339  """
340  beginOrder = kwds.pop("beginOrder", None)
341  endOrder = kwds.pop("endOrder", None)
342  for plugin in self.plugins.iterN():
343  if beginOrder is not None and plugin.getExecutionOrder() < beginOrder:
344  continue
345  if endOrder is not None and plugin.getExecutionOrder() >= endOrder:
346  break
347  self.doMeasurementN(plugin, measCat, *args, **kwds)
348 
349  def doMeasurementN(self, plugin, measCat, *args, **kwds):
350  """!
351  Call the measureN() method on the nominated plugin, handling exceptions in a consistent way.
352 
353  @param[in] plugin Plugin that will measure
354  @param[in,out] measCat lsst.afw.table.SourceCatalog containing records for just
355  the source family to be measured, and where outputs should
356  be written.
357  @param[in] *args Positional arguments forwarded to plugin.measureN()
358  @param[in] **kwds Keyword arguments forwarded to plugin.measureN()
359 
360  This method can be used with plugins that have different signatures; the only requirement is that
361  the 'plugin' and 'measCat' be the first two arguments. Subsequent positional arguments and
362  keyword arguments are forwarded directly to the plugin.
363 
364  This method should be considered "protected"; it is intended for use by derived classes, not users.
365  """
366  try:
367  plugin.measureN(measCat, *args, **kwds)
368  except FATAL_EXCEPTIONS:
369  raise
370 
371  except MeasurementError as error:
372  for measRecord in measCat:
373  lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).debug(
374  "MeasurementError in %s.measureN on records %s-%s: %s"
375  % (plugin.name, measCat[0].getId(), measCat[-1].getId(), error))
376  plugin.fail(measRecord, error)
377  except Exception as error:
378  for measRecord in measCat:
379  plugin.fail(measRecord)
380  lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).debug(
381  "Exception in %s.measureN on records %s-%s: %s"
382  % (plugin.name, measCat[0].getId(), measCat[-1].getId(), error))
Base config class for all measurement plugins.
def callMeasure(self, measRecord, args, kwds)
Call the measure() method on all plugins, handling exceptions in a consistent way.
def __init__(self, algMetadata=None, kwds)
Constructor; only called by derived classes.
def callMeasureN(self, measCat, args, kwds)
Call the measureN() method on all plugins, handling exceptions in a consistent way.
Base class for measurement plugins.
Definition: pluginsBase.py:40
Base class measurement Plugin config classes.
Definition: pluginsBase.py:31
def doMeasurementN(self, plugin, measCat, args, kwds)
Call the measureN() method on the nominated plugin, handling exceptions in a consistent way...
Map of plugins (instances of subclasses of BasePlugin) to be run for a task.
Slot configuration which assigns a particular named plugin to each of a set of slots.
static Log getLogger(std::string const &loggername)
Ultimate base class for all measurement tasks.
Base config class for all measurement driver tasks.
def doMeasurement(self, plugin, measRecord, args, kwds)
Call the measure() method on the nominated plugin, handling exceptions in a consistent way...