lsst.meas.base  14.0
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Macros Groups Pages
baseMeasurement.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 #
3 # LSST Data Management System
4 # Copyright 2008-2016 AURA/LSST.
5 #
6 # This product includes software developed by the
7 # LSST Project (http://www.lsst.org/).
8 #
9 # This program is free software: you can redistribute it and/or modify
10 # it under the terms of the GNU General Public License as published by
11 # the Free Software Foundation, either version 3 of the License, or
12 # (at your option) any later version.
13 #
14 # This program is distributed in the hope that it will be useful,
15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 # GNU General Public License for more details.
18 #
19 # You should have received a copy of the LSST License Statement and
20 # the GNU General Public License along with this program. If not,
21 # see <https://www.lsstcorp.org/LegalNotices/>.
22 #
23 """Base measurement task, which subclassed by the single frame and forced measurement tasks.
24 """
25 import lsst.pipe.base
26 import lsst.pex.config
27 
28 from .pluginRegistry import PluginMap
29 from .exceptions import FatalAlgorithmError, MeasurementError
30 from .pluginsBase import BasePluginConfig, BasePlugin
31 from .noiseReplacer import NoiseReplacerConfig
32 
33 __all__ = ("BaseMeasurementPluginConfig", "BaseMeasurementPlugin",
34  "BaseMeasurementConfig", "BaseMeasurementTask")
35 
36 # Exceptions that the measurement tasks should always propagate up to their callers
37 FATAL_EXCEPTIONS = (MemoryError, FatalAlgorithmError)
38 
39 
40 class BaseMeasurementPluginConfig(BasePluginConfig):
41  """!
42  Base config class for all measurement plugins
43 
44  Most derived classes will want to override setDefaults() in order to customize
45  the default exceutionOrder.
46 
47  A derived class whose corresponding Plugin class implements measureN() should
48  additionally add a bool doMeasureN field to replace the bool class attribute
49  defined here.
50  """
51 
52  doMeasure = lsst.pex.config.Field(dtype=bool, default=True,
53  doc="whether to run this plugin in single-object mode")
54 
55  doMeasureN = False # replace this class attribute with a Field if measureN-capable
56 
57 
58 class BaseMeasurementPlugin(BasePlugin):
59  '''
60  Base class for all measurement plugins
61 
62  This is class is a placeholder for future behavior which will be shared only between
63  measurement plugins and is implemented for symmetry with the measurement base plugin
64  configuration class
65  '''
66  pass
67 
68 
69 class SourceSlotConfig(lsst.pex.config.Config):
70  """!
71  Slot configuration which assigns a particular named plugin to each of a set of
72  slots. Each slot allows a type of measurement to be fetched from the SourceTable
73  without knowing which algorithm was used to produced the data.
74 
75  NOTE: the default algorithm for each slot must be registered, even if the default is not used.
76  """
77 
78  centroid = lsst.pex.config.Field(dtype=str, default="base_SdssCentroid", optional=True,
79  doc="the name of the centroiding algorithm used to set source x,y")
80  shape = lsst.pex.config.Field(dtype=str, default="base_SdssShape", optional=True,
81  doc="the name of the algorithm used to set source moments parameters")
82  apFlux = lsst.pex.config.Field(dtype=str, default="base_CircularApertureFlux_12_0", optional=True,
83  doc="the name of the algorithm used to set the source aperture flux slot")
84  modelFlux = lsst.pex.config.Field(dtype=str, default="base_GaussianFlux", optional=True,
85  doc="the name of the algorithm used to set the source model flux slot")
86  psfFlux = lsst.pex.config.Field(dtype=str, default="base_PsfFlux", optional=True,
87  doc="the name of the algorithm used to set the source psf flux slot")
88  instFlux = lsst.pex.config.Field(dtype=str, default="base_GaussianFlux", optional=True,
89  doc="the name of the algorithm used to set the source inst flux slot")
90  calibFlux = lsst.pex.config.Field(dtype=str, default="base_CircularApertureFlux_12_0", optional=True,
91  doc="the name of the flux measurement algorithm used for calibration")
92 
93  def setupSchema(self, schema):
94  """Convenience method to setup a Schema's slots according to the config definition.
95 
96  This is defined in the Config class to support use in unit tests without needing
97  to construct a Task object.
98  """
99  aliases = schema.getAliasMap()
100  if self.centroid is not None:
101  aliases.set("slot_Centroid", self.centroid)
102  if self.shape is not None:
103  aliases.set("slot_Shape", self.shape)
104  if self.apFlux is not None:
105  aliases.set("slot_ApFlux", self.apFlux)
106  if self.modelFlux is not None:
107  aliases.set("slot_ModelFlux", self.modelFlux)
108  if self.psfFlux is not None:
109  aliases.set("slot_PsfFlux", self.psfFlux)
110  if self.instFlux is not None:
111  aliases.set("slot_InstFlux", self.instFlux)
112  if self.calibFlux is not None:
113  aliases.set("slot_CalibFlux", self.calibFlux)
114 
115 
116 class BaseMeasurementConfig(lsst.pex.config.Config):
117  """!
118  Base config class for all measurement driver tasks.
119 
120  Subclasses should define the 'plugins' and 'undeblended' registries, e.g.:
121 
122  plugins = PluginBaseClass.registry.makeField(
123  multi=True,
124  default=[],
125  doc="Plugins to be run and their configuration"
126  )
127  undeblended = PluginBaseClass.registry.makeField(
128  multi=True,
129  default=[],
130  doc="Plugins to run on undeblended image"
131  )
132 
133  where PluginBaseClass is the appropriate base class of the plugin
134  (e.g., SingleFramePlugin or ForcedPlugin).
135  """
136 
137  slots = lsst.pex.config.ConfigField(
138  dtype=SourceSlotConfig,
139  doc="Mapping from algorithms to special aliases in Source."
140  )
141 
142  doReplaceWithNoise = lsst.pex.config.Field(
143  dtype=bool, default=True, optional=False,
144  doc='When measuring, replace other detected footprints with noise?')
145 
146  noiseReplacer = lsst.pex.config.ConfigField(
147  dtype=NoiseReplacerConfig,
148  doc="configuration that sets how to replace neighboring sources with noise"
149  )
150  undeblendedPrefix = lsst.pex.config.Field(
151  dtype=str, default="undeblended_",
152  doc="Prefix to give undeblended plugins"
153  )
154 
155  def validate(self):
156  lsst.pex.config.Config.validate(self)
157  if self.slots.centroid is not None and self.slots.centroid not in self.plugins.names:
158  raise ValueError("source centroid slot algorithm is not being run.")
159  if self.slots.shape is not None and self.slots.shape not in self.plugins.names:
160  raise ValueError("source shape slot algorithm '%s' is not being run." % self.slots.shape)
161  for slot in (self.slots.psfFlux, self.slots.apFlux, self.slots.modelFlux,
162  self.slots.instFlux, self.slots.calibFlux):
163  if slot is not None:
164  for name in self.plugins.names:
165  if len(name) <= len(slot) and name == slot[:len(name)]:
166  break
167  else:
168  raise ValueError("source flux slot algorithm '%s' is not being run." % slot)
169 
170 ## @addtogroup LSST_task_documentation
171 ## @{
172 ## @page baseMeasurementTask
173 ## BaseMeasurementTask @copybrief BaseMeasurementTask
174 ## @}
175 
176 
177 class BaseMeasurementTask(lsst.pipe.base.Task):
178  """!
179  Ultimate base class for all measurement tasks.
180 
181  This base class for SingleFrameMeasurementTask and ForcedMeasurementTask mostly exists to share
182  code between the two, and generally should not be used directly.
183  """
184 
185  ConfigClass = BaseMeasurementConfig
186  _DefaultName = "measurement"
187 
188  def __init__(self, algMetadata=None, **kwds):
189  """!
190  Constructor; only called by derived classes.
191 
192  @param[in] algMetadata An lsst.daf.base.PropertyList that will be filled with metadata
193  about the plugins being run. If None, an empty PropertyList will
194  be created.
195  @param[in] **kwds Additional arguments passed to lsst.pipe.base.Task.__init__.
196 
197  This attaches two public attributes to the class for use by derived classes and parent tasks:
198  - plugins: an empty PluginMap, which will eventually contain all active plugins that will by
199  invoked by the run() method (to be filled by subclasses). This should be considered read-only.
200  - algMetadata: a lsst.daf.base.PropertyList that will contain additional information about the
201  active plugins to be saved with the output catalog (to be filled by subclasses).
202  """
203  super(BaseMeasurementTask, self).__init__(**kwds)
204  self.plugins = PluginMap()
205  self.undeblendedPlugins = PluginMap()
206  if algMetadata is None:
207  algMetadata = lsst.daf.base.PropertyList()
208  self.algMetadata = algMetadata
209 
210  def getPluginLogName(self, pluginName):
211  return self.log.getName() + '.' + pluginName
212 
213  def initializePlugins(self, **kwds):
214  """Initialize the plugins (and slots) according to the configuration.
215 
216  Derived class constructors should call this method to fill the self.plugins
217  attribute and add correspond output fields and slot aliases to the output schema.
218 
219  In addition to the attributes added by BaseMeasurementTask.__init__, a self.schema
220  attribute holding the output schema must also be present before this method is called, .
221 
222  Keyword arguments are forwarded directly to plugin constructors, allowing derived
223  classes to use plugins with different signatures.
224  """
225  # Make a place at the beginning for the centroid plugin to run first (because it's an OrderedDict,
226  # adding an empty element in advance means it will get run first when it's reassigned to the
227  # actual Plugin).
228  if self.config.slots.centroid is not None:
229  self.plugins[self.config.slots.centroid] = None
230  # Init the plugins, sorted by execution order. At the same time add to the schema
231  for executionOrder, name, config, PluginClass in sorted(self.config.plugins.apply()):
232  # Pass logName to the plugin if the plugin is marked as using it
233  # The task will use this name to log plugin errors, regardless.
234  if hasattr(PluginClass, "hasLogName") and PluginClass.hasLogName:
235  self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata,
236  logName=self.getPluginLogName(name), **kwds)
237  else:
238  self.plugins[name] = PluginClass(config, name, metadata=self.algMetadata, **kwds)
239 
240  # In rare circumstances (usually tests), the centroid slot not be coming from an algorithm,
241  # which means we'll have added something we don't want to the plugins map, and we should
242  # remove it.
243  if self.config.slots.centroid is not None and self.plugins[self.config.slots.centroid] is None:
244  del self.plugins[self.config.slots.centroid]
245  # Initialize the plugins to run on the undeblended image
246  for executionOrder, name, config, PluginClass in sorted(self.config.undeblended.apply()):
247  undeblendedName = self.config.undeblendedPrefix + name
248  self.undeblendedPlugins[name] = PluginClass(config, undeblendedName, metadata=self.algMetadata,
249  **kwds)
250 
251  def callMeasure(self, measRecord, *args, **kwds):
252  """!
253  Call the measure() method on all plugins, handling exceptions in a consistent way.
254 
255  @param[in,out] measRecord lsst.afw.table.SourceRecord that corresponds to the object being
256  measured, and where outputs should be written.
257  @param[in] *args Positional arguments forwarded to Plugin.measure()
258  @param[in] **kwds Keyword arguments. Two are handled locally:
259  - beginOrder: beginning execution order (inclusive): measurements with
260  executionOrder < beginOrder are not executed. None for no limit.
261  - endOrder: ending execution order (exclusive): measurements with
262  executionOrder >= endOrder are not executed. None for no limit.
263  the rest are forwarded to Plugin.measure()
264 
265  This method can be used with plugins that have different signatures; the only requirement is that
266  'measRecord' be the first argument. Subsequent positional arguments and keyword arguments are
267  forwarded directly to the plugin.
268 
269  This method should be considered "protected"; it is intended for use by derived classes, not users.
270  """
271  beginOrder = kwds.pop("beginOrder", None)
272  endOrder = kwds.pop("endOrder", None)
273  for plugin in self.plugins.iter():
274  if beginOrder is not None and plugin.getExecutionOrder() < beginOrder:
275  continue
276  if endOrder is not None and plugin.getExecutionOrder() >= endOrder:
277  break
278  self.doMeasurement(plugin, measRecord, *args, **kwds)
279 
280  def doMeasurement(self, plugin, measRecord, *args, **kwds):
281  """!
282  Call the measure() method on the nominated plugin, handling exceptions in a consistent way.
283 
284  @param[in] plugin Plugin that will measure
285  @param[in,out] measRecord lsst.afw.table.SourceRecord that corresponds to the object being
286  measured, and where outputs should be written.
287  @param[in] *args Positional arguments forwarded to plugin.measure()
288  @param[in] **kwds Keyword arguments forwarded to plugin.measure()
289 
290  This method can be used with plugins that have different signatures; the only requirement is that
291  the 'plugin' and 'measRecord' be the first two arguments. Subsequent positional arguments and
292  keyword arguments are forwarded directly to the plugin.
293 
294  This method should be considered "protected"; it is intended for use by derived classes, not users.
295  """
296  try:
297  plugin.measure(measRecord, *args, **kwds)
298  except FATAL_EXCEPTIONS:
299  raise
300  except MeasurementError as error:
301  lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).debug(
302  "MeasurementError in %s.measure on record %s: %s"
303  % (plugin.name, measRecord.getId(), error))
304  plugin.fail(measRecord, error)
305  except Exception as error:
306  lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).debug(
307  "Exception in %s.measure on record %s: %s"
308  % (plugin.name, measRecord.getId(), error))
309  plugin.fail(measRecord)
310 
311  def callMeasureN(self, measCat, *args, **kwds):
312  """!
313  Call the measureN() method on all plugins, handling exceptions in a consistent way.
314 
315  @param[in,out] measCat lsst.afw.table.SourceCatalog containing records for just
316  the source family to be measured, and where outputs should
317  be written.
318  @param[in] beginOrder beginning execution order (inclusive): measurements with
319  executionOrder < beginOrder are not executed. None for no limit.
320  @param[in] endOrder ending execution order (exclusive): measurements with
321  executionOrder >= endOrder are not executed. None for no limit.
322  @param[in] *args Positional arguments forwarded to Plugin.measure()
323  @param[in] **kwds Keyword arguments. Two are handled locally:
324  - beginOrder: beginning execution order (inclusive): measurements with
325  executionOrder < beginOrder are not executed. None for no limit.
326  - endOrder: ending execution order (exclusive): measurements with
327  executionOrder >= endOrder are not executed. None for no limit.
328  the rest are forwarded to Plugin.measure()
329 
330  This method can be used with plugins that have different signatures; the only requirement is that
331  'measRecord' be the first argument. Subsequent positional arguments and keyword arguments are
332  forwarded directly to the plugin.
333 
334  This method should be considered "protected"; it is intended for use by derived classes, not users.
335  """
336  beginOrder = kwds.pop("beginOrder", None)
337  endOrder = kwds.pop("endOrder", None)
338  for plugin in self.plugins.iterN():
339  if beginOrder is not None and plugin.getExecutionOrder() < beginOrder:
340  continue
341  if endOrder is not None and plugin.getExecutionOrder() >= endOrder:
342  break
343  self.doMeasurementN(plugin, measCat, *args, **kwds)
344 
345  def doMeasurementN(self, plugin, measCat, *args, **kwds):
346  """!
347  Call the measureN() method on the nominated plugin, handling exceptions in a consistent way.
348 
349  @param[in] plugin Plugin that will measure
350  @param[in,out] measCat lsst.afw.table.SourceCatalog containing records for just
351  the source family to be measured, and where outputs should
352  be written.
353  @param[in] *args Positional arguments forwarded to plugin.measureN()
354  @param[in] **kwds Keyword arguments forwarded to plugin.measureN()
355 
356  This method can be used with plugins that have different signatures; the only requirement is that
357  the 'plugin' and 'measCat' be the first two arguments. Subsequent positional arguments and
358  keyword arguments are forwarded directly to the plugin.
359 
360  This method should be considered "protected"; it is intended for use by derived classes, not users.
361  """
362  try:
363  plugin.measureN(measCat, *args, **kwds)
364  except FATAL_EXCEPTIONS:
365  raise
366 
367  except MeasurementError as error:
368  for measRecord in measCat:
369  lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).debug(
370  "MeasurementError in %s.measureN on records %s-%s: %s"
371  % (plugin.name, measCat[0].getId(), measCat[-1].getId(), error))
372  plugin.fail(measRecord, error)
373  except Exception as error:
374  for measRecord in measCat:
375  plugin.fail(measRecord)
376  lsst.log.Log.getLogger(self.getPluginLogName(plugin.name)).debug(
377  "Exception in %s.measureN on records %s-%s: %s"
378  % (plugin.name, measCat[0].getId(), measCat[-1].getId(), error))
Base config class for all measurement plugins.
def __init__
Constructor; only called by derived classes.
def callMeasure
Call the measure() method on all plugins, handling exceptions in a consistent way.
def doMeasurement
Call the measure() method on the nominated plugin, handling exceptions in a consistent way...
def doMeasurementN
Call the measureN() method on the nominated plugin, handling exceptions in a consistent way...
Ultimate base class for all measurement tasks.
Base config class for all measurement driver tasks.
def callMeasureN
Call the measureN() method on all plugins, handling exceptions in a consistent way.