lsst.meas.base  16.0-12-g5ad1ebf+6
catalogCalculation.py
Go to the documentation of this file.
1 from collections import namedtuple
2 
3 import lsst.pipe.base
4 import lsst.pex.config
5 import lsst.daf.base
6 
7 from .pluginsBase import BasePlugin, BasePluginConfig
8 from .pluginRegistry import PluginRegistry, PluginMap
9 from . import FatalAlgorithmError, MeasurementError
10 
11 # Exceptions that the measurement tasks should always propagate up to their callers
12 FATAL_EXCEPTIONS = (MemoryError, FatalAlgorithmError)
13 
14 __all__ = ("CatalogCalculationPluginConfig", "CatalogCalculationPlugin", "CatalogCalculationConfig",
15  "CatalogCalculationTask")
16 
17 
19  '''
20  Default configuration class for catalogCalcuation plugins
21  '''
22  pass
23 
24 
26  '''
27  Base class for after CatalogCalculation plugin
28  '''
29  registry = PluginRegistry(CatalogCalculationPluginConfig)
30  ConfigClass = CatalogCalculationPluginConfig
31  # This defines if the plugin operates on a single source at a time, or expects the whole catalog.
32  # The value defaults to single for a single source, set to multi when the plugin expects the whole
33  # catalog. If The plugin is of type multi, the fail method should be implemented to accept the whole
34  # catalog. If the plugin is of type the fail method should accept a single source record.
35 
36  plugType = 'single'
37 
38  def __init__(self, config, name, schema, metadata):
39  """!
40  Initialize the catalogCalculation plugin
41 
42  @param[in] config An instance of catalogCalculation config class.
43  @param[in] name The string the plugin was registered with.
44  @param[in,out] schema The source schema, New fields should be added here to
45  hold output produced by this plugin.
46  @param[in] metadata Plugin metadata that will be attached to the output catalog
47  """
48  BasePlugin.__init__(self, config, name)
49 
50  @classmethod
52  ''' Sets the relative order of plugins (smaller numbers run first).
53 
54  CatalogCalculation plugins must run with BasePlugin.DEFAULT_CATALOGCALCULATION or higher
55 
56  All plugins must implement this method with an appropriate run level
57  '''
58  raise NotImplementedError()
59 
60  def calculate(self, cat, **kwargs):
61  """!
62  Process either a single catalog enter or the whole catalog and produce output defined by the plugin
63 
64  @param[in,out] cat Either a lsst source catalog or a catalog entery depending on the plug type
65  specified in the classes configuration. Results may be added to new columns,
66  or existing entries altered.
67  @param[in] kwargs Any additional kwargs that may be passed through the CatalogCalculationPlugin.
68  """
69  raise NotImplementedError()
70 
71 
72 class CCContext:
73  '''
74  Context manager to handle catching errors that may have been thrown in a catalogCalculation plugin
75  @param[in] plugin The plugin that is to be run
76  @param[in] cat Either a catalog or a source record entry of a catalog, depending of the plugin type,
77  i.e. either working on a whole catalog, or a single record.
78  @param[in] log The log which to write to, most likely will always be the log (self.log) of the object
79  in which the context manager is used.
80  '''
81 
82  def __init__(self, plugin, cat, log):
83  self.plugin = plugin
84  self.cat = cat
85  self.log = log
86 
87  def __enter__(self):
88  return
89 
90  def __exit__(self, exc_type, exc_value, traceback):
91  if exc_type is None:
92  return True
93  if exc_type in FATAL_EXCEPTIONS:
94  raise exc_value
95  elif exc_type is MeasurementError:
96  self.plugin.fail(self.cat, exc_value)
97  else:
98  self.log.warn("Error in {}.calculate: {}".format(self.plugin.name, exc_value))
99  return True
100 
101 
102 class CatalogCalculationConfig(lsst.pex.config.Config):
103  '''
104  Config class for catalog calculation driver task.
105 
106  Specifies which plugins will execute when CatalogCalculationTask
107  associated with this configuration is run.
108  '''
109  plugins = CatalogCalculationPlugin.registry.makeField(
110  multi=True,
111  default=["base_ClassificationExtendedness",
112  "base_FootprintArea"],
113  doc="Plugins to be run and their configuration")
114 
115 
116 class CatalogCalculationTask(lsst.pipe.base.Task):
117  '''
118  This task facilitates running plugins which will operate on a source catalog. These plugins may do things
119  such as classifying an object based on source record entries inserted during a measurement task.
120 
121  Plugins may either take an entire catalog to work on at a time, or work on individual records
122  '''
123  ConfigClass = CatalogCalculationConfig
124  _DefaultName = "catalogCalculation"
125 
126  def __init__(self, schema, plugMetadata=None, **kwargs):
127  """
128  Constructor; only called by derived classes
129 
130  @param[in] plugMetaData An lsst.daf.base.PropertyList that will be filled with metadata
131  about the plugins being run. If None, an empty empty PropertyList
132  will be created.
133  @param[in] **kwargs Additional arguments passed to lsst.pipe.base.Task.__init__.
134  """
135  lsst.pipe.base.Task.__init__(self, **kwargs)
136  self.schema = schema
137  if plugMetadata is None:
138  plugMetadata = lsst.daf.base.PropertyList()
139  self.plugMetadata = plugMetadata
141 
142  self.initializePlugins()
143 
144  def initializePlugins(self):
145  '''
146  Initialize the plugins according to the configuration.
147  '''
148 
149  pluginType = namedtuple('pluginType', 'single multi')
150  self.executionDict = {}
151  # Read the properties for each plugin. Allocate a dictionary entry for each run level. Verify that
152  # the plugins are above the minimum run level for an catalogCalculation plugin. For each run level,
153  # the plugins are sorted into either single record, or multi record groups to later be run
154  # appropriately
155  for executionOrder, name, config, PluginClass in sorted(self.config.plugins.apply()):
156  if executionOrder not in self.executionDict:
157  self.executionDict[executionOrder] = pluginType(single=[], multi=[])
158  if PluginClass.getExecutionOrder() >= BasePlugin.DEFAULT_CATALOGCALCULATION:
159  plug = PluginClass(config, name, self.schema, metadata=self.plugMetadata)
160  self.plugins[name] = plug
161  if plug.plugType == 'single':
162  self.executionDict[executionOrder].single.append(plug)
163  elif plug.plugType == 'multi':
164  self.executionDict[executionOrder].multi.append(plug)
165  else:
166  errorTuple = (PluginClass, PluginClass.getExecutionOrder(),
167  BasePlugin.DEFAULT_CATALOGCALCULATION)
168  raise ValueError("{} has an execution order less than the minimum for an catalogCalculation "
169  "plugin. Value {} : Minimum {}".format(*errorTuple))
170 
171  @lsst.pipe.base.timeMethod
172  def run(self, measCat):
173  '''
174  The entry point for the catalogCalculation task. This method should be called with a reference to a
175  measurement catalog.
176  '''
177  self.callCompute(measCat)
178 
179  def callCompute(self, catalog):
180  '''
181  Run each of the plugins on the catalog
182  @param[in] catalog The catalog on which the plugins will operate
183  '''
184  for runlevel in sorted(self.executionDict):
185  # Run all of the plugins which take a whole catalog first
186  for plug in self.executionDict[runlevel].multi:
187  with CCContext(plug, catalog, self.log):
188  plug.calculate(catalog)
189  # Run all the plugins which take single catalog entries
190  for measRecord in catalog:
191  for plug in self.executionDict[runlevel].single:
192  with CCContext(plug, measRecord, self.log):
193  plug.calculate(measRecord)
def __exit__(self, exc_type, exc_value, traceback)
Base class for measurement plugins.
Definition: pluginsBase.py:40
def __init__(self, schema, plugMetadata=None, kwargs)
Base class measurement Plugin config classes.
Definition: pluginsBase.py:31
Base class for plugin registries.
Map of plugins (instances of subclasses of BasePlugin) to be run for a task.
def __init__(self, config, name, schema, metadata)
Initialize the catalogCalculation plugin.
def calculate(self, cat, kwargs)
Process either a single catalog enter or the whole catalog and produce output defined by the plugin...