lsst.meas.base  16.0-17-g7e0e4ff+13
baseMeasurement.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2016 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 """Base measurement task, which subclassed by the single frame and forced measurement tasks.
24 """
25 import lsst.pipe.base
26 import lsst.pex.config
27 
28 from .pluginRegistry import PluginMap
29 from .exceptions import FatalAlgorithmError, MeasurementError
30 from .pluginsBase import BasePluginConfig, BasePlugin
31 from .noiseReplacer import NoiseReplacerConfig
32 
33 __all__ = ("BaseMeasurementPluginConfig", "BaseMeasurementPlugin",
34  "BaseMeasurementConfig", "BaseMeasurementTask")
35 
36 # Exceptions that the measurement tasks should always propagate up to their callers
37 FATAL_EXCEPTIONS = (MemoryError, FatalAlgorithmError)
38 
39 
41  """!
42  Base config class for all measurement plugins
43 
44  Most derived classes will want to override setDefaults() in order to customize
45  the default exceutionOrder.
46 
47  A derived class whose corresponding Plugin class implements measureN() should
48  additionally add a bool doMeasureN field to replace the bool class attribute
49  defined here.
50  """
51 
52  doMeasure = lsst.pex.config.Field(dtype=bool, default=True,
53  doc="whether to run this plugin in single-object mode")
54 
55  doMeasureN = False # replace this class attribute with a Field if measureN-capable
56 
57 
59  '''
60  Base class for all measurement plugins
61 
62  This is class is a placeholder for future behavior which will be shared only between
63  measurement plugins and is implemented for symmetry with the measurement base plugin
64  configuration class
65  '''
66  pass
67 
68 
69 class SourceSlotConfig(lsst.pex.config.Config):
70  """!
71  Slot configuration which assigns a particular named plugin to each of a set of
72  slots. Each slot allows a type of measurement to be fetched from the SourceTable
73  without knowing which algorithm was used to produced the data.
74 
75  NOTE: the default algorithm for each slot must be registered, even if the default is not used.
76  """
77 
78  Field = lsst.pex.config.Field
79  centroid = Field(dtype=str, default="base_SdssCentroid", optional=True,
80  doc="the name of the centroiding algorithm used to set source x,y")
81  shape = Field(dtype=str, default="base_SdssShape", optional=True,
82  doc="the name of the algorithm used to set source moments parameters")
83  psfShape = Field(dtype=str, default="base_SdssShape_psf", optional=True,
84  doc="the name of the algorithm used to set PSF moments parameters")
85  apFlux = Field(dtype=str, default="base_CircularApertureFlux_12_0", optional=True,
86  doc="the name of the algorithm used to set the source aperture instFlux slot")
87  modelFlux = Field(dtype=str, default="base_GaussianFlux", optional=True,
88  doc="the name of the algorithm used to set the source model instFlux slot")
89  psfFlux = Field(dtype=str, default="base_PsfFlux", optional=True,
90  doc="the name of the algorithm used to set the source psf instFlux slot")
91  gaussianFlux = Field(dtype=str, default="base_GaussianFlux", optional=True,
92  doc="the name of the algorithm used to set the source Gaussian instFlux slot")
93  calibFlux = Field(dtype=str, default="base_CircularApertureFlux_12_0", optional=True,
94  doc="the name of the instFlux measurement algorithm used for calibration")
95 
96  def setupSchema(self, schema):
97  """Convenience method to setup a Schema's slots according to the config definition.
98 
99  This is defined in the Config class to support use in unit tests without needing
100  to construct a Task object.
101  """
102  aliases = schema.getAliasMap()
103  if self.centroid is not None:
104  aliases.set("slot_Centroid", self.centroid)
105  if self.shape is not None:
106  aliases.set("slot_Shape", self.shape)
107  if self.psfShape is not None:
108  aliases.set("slot_PsfShape", self.psfShape)
109  if self.apFlux is not None:
110  aliases.set("slot_ApFlux", self.apFlux)
111  if self.modelFlux is not None:
112  aliases.set("slot_ModelFlux", self.modelFlux)
113  if self.psfFlux is not None:
114  aliases.set("slot_PsfFlux", self.psfFlux)
115  if self.gaussianFlux is not None:
116  aliases.set("slot_GaussianFlux", self.gaussianFlux)
117  if self.calibFlux is not None:
118  aliases.set("slot_CalibFlux", self.calibFlux)
119 
120 
121 class BaseMeasurementConfig(lsst.pex.config.Config):
122  """!
123  Base config class for all measurement driver tasks.
124 
125  Subclasses should define the 'plugins' and 'undeblended' registries, e.g.:
126 
127  plugins = PluginBaseClass.registry.makeField(
128  multi=True,
129  default=[],
130  doc="Plugins to be run and their configuration"
131  )
132  undeblended = PluginBaseClass.registry.makeField(
133  multi=True,
134  default=[],
135  doc="Plugins to run on undeblended image"
136  )
137 
138  where PluginBaseClass is the appropriate base class of the plugin
139  (e.g., SingleFramePlugin or ForcedPlugin).
140  """
141 
142  slots = lsst.pex.config.ConfigField(
143  dtype=SourceSlotConfig,
144  doc="Mapping from algorithms to special aliases in Source."
145  )
146 
147  doReplaceWithNoise = lsst.pex.config.Field(
148  dtype=bool, default=True, optional=False,
149  doc='When measuring, replace other detected footprints with noise?')
150 
151  noiseReplacer = lsst.pex.config.ConfigField(
152  dtype=NoiseReplacerConfig,
153  doc="configuration that sets how to replace neighboring sources with noise"
154  )
155  undeblendedPrefix = lsst.pex.config.Field(
156  dtype=str, default="undeblended_",
157  doc="Prefix to give undeblended plugins"
158  )
159 
160  def validate(self):
161  lsst.pex.config.Config.validate(self)
162  if self.slots.centroid is not None and self.slots.centroid not in self.plugins.names:
163  raise ValueError("source centroid slot algorithm is not being run.")
164  if self.slots.shape is not None and self.slots.shape not in self.plugins.names:
165  raise ValueError("source shape slot algorithm '%s' is not being run." % self.slots.shape)
166  for slot in (self.slots.psfFlux, self.slots.apFlux, self.slots.modelFlux,
167  self.slots.gaussianFlux, self.slots.calibFlux):
168  if slot is not None:
169  for name in self.plugins.names:
170  if len(name) <= len(slot) and name == slot[:len(name)]:
171  break
172  else:
173  raise ValueError("source instFlux slot algorithm '%s' is not being run." % slot)
174 
175 
180 
181 
182 class BaseMeasurementTask(lsst.pipe.base.Task):
183  """!
184  Ultimate base class for all measurement tasks.
185 
186  This base class for SingleFrameMeasurementTask and ForcedMeasurementTask mostly exists to share
187  code between the two, and generally should not be used directly.
188  """
189 
190  ConfigClass = BaseMeasurementConfig
191  _DefaultName = "measurement"
192 
193  def __init__(self, algMetadata=None, **kwds):
194  """!
195  Constructor; only called by derived classes.
196 
197  @param[in] algMetadata An lsst.daf.base.PropertyList that will be filled with metadata
198  about the plugins being run. If None, an empty PropertyList will
199  be created.
200  @param[in] **kwds Additional arguments passed to lsst.pipe.base.Task.__init__.
201 
202  This attaches two public attributes to the class for use by derived classes and parent tasks:
203  - plugins: an empty PluginMap, which will eventually contain all active plugins that will by
204  invoked by the run() method (to be filled by subclasses). This should be considered read-only.
205  - algMetadata: a lsst.daf.base.PropertyList that will contain additional information about the
206  active plugins to be saved with the output catalog (to be filled by subclasses).
207  """
208  super(BaseMeasurementTask, self).__init__(**kwds)
211  if algMetadata is None:
212  algMetadata = lsst.daf.base.PropertyList()
213  self.algMetadata = algMetadata
214 
215  def getPluginLogName(self, pluginName):
216  return self.log.getName() + '.' + pluginName
217 
218  def initializePlugins(self, **kwds):
219  """Initialize the plugins (and slots) according to the configuration.
220 
221  Derived class constructors should call this method to fill the self.plugins
222  attribute and add correspond output fields and slot aliases to the output schema.
223 
224  In addition to the attributes added by BaseMeasurementTask.__init__, a self.schema
225  attribute holding the output schema must also be present before this method is called, .
226 
227  Keyword arguments are forwarded directly to plugin constructors, allowing derived
228  classes to use plugins with different signatures.
229  """
230  # Make a place at the beginning for the centroid plugin to run first (because it's an OrderedDict,
231  # adding an empty element in advance means it will get run first when it's reassigned to the
232  # actual Plugin).
233  if self.config.slots.centroid is not None:
234  self.plugins[self.config.slots.centroid] = None
235  # Init the plugins, sorted by execution order. At the same time add to the schema
236  for executionOrder, name, config, PluginClass in sorted(self.config.plugins.apply()):
237  # Pass logName to the plugin if the plugin is marked as using it
238  # The task will use this name to log plugin errors, regardless.
239  if hasattr(PluginClass, "hasLogName") and PluginClass.hasLogName:
240  self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata,
241  logName=self.getPluginLogName(name), **kwds)
242  else:
243  self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata, **kwds)
244 
245  # In rare circumstances (usually tests), the centroid slot not be coming from an algorithm,
246  # which means we'll have added something we don't want to the plugins map, and we should
247  # remove it.
248  if self.config.slots.centroid is not None and self.plugins[self.config.slots.centroid] is None:
249  del self.plugins[self.config.slots.centroid]
250  # Initialize the plugins to run on the undeblended image
251  for executionOrder, name, config, PluginClass in sorted(self.config.undeblended.apply()):
252  undeblendedName = self.config.undeblendedPrefix + name
253  self.undeblendedPlugins[name] = PluginClass(config, undeblendedName, metadata=self.algMetadata,
254  **kwds)
255 
256  def callMeasure(self, measRecord, *args, **kwds):
257  """!
258  Call the measure() method on all plugins, handling exceptions in a consistent way.
259 
260  @param[in,out] measRecord lsst.afw.table.SourceRecord that corresponds to the object being
261  measured, and where outputs should be written.
262  @param[in] *args Positional arguments forwarded to Plugin.measure()
263  @param[in] **kwds Keyword arguments. Two are handled locally:
264  - beginOrder: beginning execution order (inclusive): measurements with
265  executionOrder < beginOrder are not executed. None for no limit.
266  - endOrder: ending execution order (exclusive): measurements with
267  executionOrder >= endOrder are not executed. None for no limit.
268  the rest are forwarded to Plugin.measure()
269 
270  This method can be used with plugins that have different signatures; the only requirement is that
271  'measRecord' be the first argument. Subsequent positional arguments and keyword arguments are
272  forwarded directly to the plugin.
273 
274  This method should be considered "protected"; it is intended for use by derived classes, not users.
275  """
276  beginOrder = kwds.pop("beginOrder", None)
277  endOrder = kwds.pop("endOrder", None)
278  for plugin in self.plugins.iter():
279  if beginOrder is not None and plugin.getExecutionOrder() < beginOrder:
280  continue
281  if endOrder is not None and plugin.getExecutionOrder() >= endOrder:
282  break
283  self.doMeasurement(plugin, measRecord, *args, **kwds)
284 
285  def doMeasurement(self, plugin, measRecord, *args, **kwds):
286  """!
287  Call the measure() method on the nominated plugin, handling exceptions in a consistent way.
288 
289  @param[in] plugin Plugin that will measure
290  @param[in,out] measRecord lsst.afw.table.SourceRecord that corresponds to the object being
291  measured, and where outputs should be written.
292  @param[in] *args Positional arguments forwarded to plugin.measure()
293  @param[in] **kwds Keyword arguments forwarded to plugin.measure()
294 
295  This method can be used with plugins that have different signatures; the only requirement is that
296  the 'plugin' and 'measRecord' be the first two arguments. Subsequent positional arguments and
297  keyword arguments are forwarded directly to the plugin.
298 
299  This method should be considered "protected"; it is intended for use by derived classes, not users.
300  """
301  try:
302  plugin.measure(measRecord, *args, **kwds)
303  except FATAL_EXCEPTIONS:
304  raise
305  except MeasurementError as error:
306  lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).debug(
307  "MeasurementError in %s.measure on record %s: %s"
308  % (plugin.name, measRecord.getId(), error))
309  plugin.fail(measRecord, error)
310  except Exception as error:
311  lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).debug(
312  "Exception in %s.measure on record %s: %s"
313  % (plugin.name, measRecord.getId(), error))
314  plugin.fail(measRecord)
315 
316  def callMeasureN(self, measCat, *args, **kwds):
317  """!
318  Call the measureN() method on all plugins, handling exceptions in a consistent way.
319 
320  @param[in,out] measCat lsst.afw.table.SourceCatalog containing records for just
321  the source family to be measured, and where outputs should
322  be written.
323  @param[in] beginOrder beginning execution order (inclusive): measurements with
324  executionOrder < beginOrder are not executed. None for no limit.
325  @param[in] endOrder ending execution order (exclusive): measurements with
326  executionOrder >= endOrder are not executed. None for no limit.
327  @param[in] *args Positional arguments forwarded to Plugin.measure()
328  @param[in] **kwds Keyword arguments. Two are handled locally:
329  - beginOrder: beginning execution order (inclusive): measurements with
330  executionOrder < beginOrder are not executed. None for no limit.
331  - endOrder: ending execution order (exclusive): measurements with
332  executionOrder >= endOrder are not executed. None for no limit.
333  the rest are forwarded to Plugin.measure()
334 
335  This method can be used with plugins that have different signatures; the only requirement is that
336  'measRecord' be the first argument. Subsequent positional arguments and keyword arguments are
337  forwarded directly to the plugin.
338 
339  This method should be considered "protected"; it is intended for use by derived classes, not users.
340  """
341  beginOrder = kwds.pop("beginOrder", None)
342  endOrder = kwds.pop("endOrder", None)
343  for plugin in self.plugins.iterN():
344  if beginOrder is not None and plugin.getExecutionOrder() < beginOrder:
345  continue
346  if endOrder is not None and plugin.getExecutionOrder() >= endOrder:
347  break
348  self.doMeasurementN(plugin, measCat, *args, **kwds)
349 
350  def doMeasurementN(self, plugin, measCat, *args, **kwds):
351  """!
352  Call the measureN() method on the nominated plugin, handling exceptions in a consistent way.
353 
354  @param[in] plugin Plugin that will measure
355  @param[in,out] measCat lsst.afw.table.SourceCatalog containing records for just
356  the source family to be measured, and where outputs should
357  be written.
358  @param[in] *args Positional arguments forwarded to plugin.measureN()
359  @param[in] **kwds Keyword arguments forwarded to plugin.measureN()
360 
361  This method can be used with plugins that have different signatures; the only requirement is that
362  the 'plugin' and 'measCat' be the first two arguments. Subsequent positional arguments and
363  keyword arguments are forwarded directly to the plugin.
364 
365  This method should be considered "protected"; it is intended for use by derived classes, not users.
366  """
367  try:
368  plugin.measureN(measCat, *args, **kwds)
369  except FATAL_EXCEPTIONS:
370  raise
371 
372  except MeasurementError as error:
373  for measRecord in measCat:
374  lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).debug(
375  "MeasurementError in %s.measureN on records %s-%s: %s"
376  % (plugin.name, measCat[0].getId(), measCat[-1].getId(), error))
377  plugin.fail(measRecord, error)
378  except Exception as error:
379  for measRecord in measCat:
380  plugin.fail(measRecord)
381  lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).debug(
382  "Exception in %s.measureN on records %s-%s: %s"
383  % (plugin.name, measCat[0].getId(), measCat[-1].getId(), error))
Base config class for all measurement plugins.
def callMeasure(self, measRecord, args, kwds)
Call the measure() method on all plugins, handling exceptions in a consistent way.
def __init__(self, algMetadata=None, kwds)
Constructor; only called by derived classes.
def callMeasureN(self, measCat, args, kwds)
Call the measureN() method on all plugins, handling exceptions in a consistent way.
static Log getLogger(std::string const &loggername)
Base class for measurement plugins.
Definition: pluginsBase.py:40
Base class measurement Plugin config classes.
Definition: pluginsBase.py:31
def doMeasurementN(self, plugin, measCat, args, kwds)
Call the measureN() method on the nominated plugin, handling exceptions in a consistent way...
Map of plugins (instances of subclasses of BasePlugin) to be run for a task.
Slot configuration which assigns a particular named plugin to each of a set of slots.
Ultimate base class for all measurement tasks.
Base config class for all measurement driver tasks.
def doMeasurement(self, plugin, measRecord, args, kwds)
Call the measure() method on the nominated plugin, handling exceptions in a consistent way...