lsst.meas.base  13.0-33-gb1a1d47+3
catalogCalculation.py
Go to the documentation of this file.
1 from collections import namedtuple
2 
3 from builtins import object
4 
5 import lsst.pipe.base
6 import lsst.pex.config
7 import lsst.daf.base
8 
9 from .pluginsBase import BasePlugin, BasePluginConfig
10 from .pluginRegistry import PluginRegistry, PluginMap
11 from . import FatalAlgorithmError, MeasurementError
12 
13 # Exceptions that the measurement tasks should always propagate up to their callers
14 FATAL_EXCEPTIONS = (MemoryError, FatalAlgorithmError)
15 
16 __all__ = ("CatalogCalculationPluginConfig", "CatalogCalculationPlugin", "CatalogCalculationConfig",
17  "CatalogCalculationTask")
18 
19 
21  '''
22  Default configuration class for catalogCalcuation plugins
23  '''
24  pass
25 
26 
28  '''
29  Base class for after CatalogCalculation plugin
30  '''
31  registry = PluginRegistry(CatalogCalculationPluginConfig)
32  ConfigClass = CatalogCalculationPluginConfig
33  # This defines if the plugin operates on a single source at a time, or expects the whole catalog.
34  # The value defaults to single for a single source, set to multi when the plugin expects the whole
35  # catalog. If The plugin is of type multi, the fail method should be implemented to accept the whole
36  # catalog. If the plugin is of type the fail method should accept a single source record.
37 
38  plugType = 'single'
39 
40  def __init__(self, config, name, schema, metadata):
41  """!
42  Initialize the catalogCalculation plugin
43 
44  @param[in] config An instance of catalogCalculation config class.
45  @param[in] name The string the plugin was registered with.
46  @param[in,out] schema The source schema, New fields should be added here to
47  hold output produced by this plugin.
48  @param[in] metadata Plugin metadata that will be attached to the output catalog
49  """
50  BasePlugin.__init__(self, config, name)
51 
52  @classmethod
54  ''' Sets the relative order of plugins (smaller numbers run first).
55 
56  CatalogCalculation plugins must run with BasePlugin.DEFAULT_CATALOGCALCULATION or higher
57 
58  All plugins must implement this method with an appropriate run level
59  '''
60  raise NotImplementedError()
61 
62  def calculate(self, cat, **kwargs):
63  """!
64  Process either a single catalog enter or the whole catalog and produce output defined by the plugin
65 
66  @param[in,out] cat Either a lsst source catalog or a catalog entery depending on the plug type
67  specified in the classes configuration. Results may be added to new columns,
68  or existing entries altered.
69  @param[in] kwargs Any additional kwargs that may be passed through the CatalogCalculationPlugin.
70  """
71  raise NotImplementedError()
72 
73 
74 class CCContext(object):
75  '''
76  Context manager to handle catching errors that may have been thrown in a catalogCalculation plugin
77  @param[in] plugin The plugin that is to be run
78  @param[in] cat Either a catalog or a source record entry of a catalog, depending of the plugin type,
79  i.e. either working on a whole catalog, or a single record.
80  @param[in] log The log which to write to, most likely will always be the log (self.log) of the object
81  in which the context manager is used.
82  '''
83 
84  def __init__(self, plugin, cat, log):
85  self.plugin = plugin
86  self.cat = cat
87  self.log = log
88 
89  def __enter__(self):
90  return
91 
92  def __exit__(self, exc_type, exc_value, traceback):
93  if exc_type is None:
94  return True
95  if exc_type in FATAL_EXCEPTIONS:
96  raise exc_value
97  elif exc_type is MeasurementError:
98  self.plugin.fail(self.cat, exc_value)
99  else:
100  self.log.warn("Error in {}.calculate: {}".format(self.plugin.name, exc_value))
101  return True
102 
103 
104 class CatalogCalculationConfig(lsst.pex.config.Config):
105  '''
106  Config class for catalog calculation driver task.
107 
108  Specifies which plugins will execute when CatalogCalculationTask
109  associated with this configuration is run.
110  '''
111  plugins = CatalogCalculationPlugin.registry.makeField(
112  multi=True,
113  default=["base_ClassificationExtendedness",
114  "base_FootprintArea"],
115  doc="Plugins to be run and their configuration")
116 
117 
118 class CatalogCalculationTask(lsst.pipe.base.Task):
119  '''
120  This task facilitates running plugins which will operate on a source catalog. These plugins may do things
121  such as classifying an object based on source record entries inserted during a measurement task.
122 
123  Plugins may either take an entire catalog to work on at a time, or work on individual records
124  '''
125  ConfigClass = CatalogCalculationConfig
126  _DefaultName = "catalogCalculation"
127 
128  def __init__(self, schema, plugMetadata=None, **kwargs):
129  """
130  Constructor; only called by derived classes
131 
132  @param[in] plugMetaData An lsst.daf.base.PropertyList that will be filled with metadata
133  about the plugins being run. If None, an empty empty PropertyList
134  will be created.
135  @param[in] **kwargs Additional arguments passed to lsst.pipe.base.Task.__init__.
136  """
137  lsst.pipe.base.Task.__init__(self, **kwargs)
138  self.schema = schema
139  if plugMetadata is None:
140  plugMetadata = lsst.daf.base.PropertyList()
141  self.plugMetadata = plugMetadata
143 
144  self.initializePlugins()
145 
146  def initializePlugins(self):
147  '''
148  Initialize the plugins according to the configuration.
149  '''
150 
151  pluginType = namedtuple('pluginType', 'single multi')
152  self.executionDict = {}
153  # Read the properties for each plugin. Allocate a dictionary entry for each run level. Verify that
154  # the plugins are above the minimum run level for an catalogCalculation plugin. For each run level,
155  # the plugins are sorted into either single record, or multi record groups to later be run
156  # appropriately
157  for executionOrder, name, config, PluginClass in sorted(self.config.plugins.apply()):
158  if executionOrder not in self.executionDict:
159  self.executionDict[executionOrder] = pluginType(single=[], multi=[])
160  if PluginClass.getExecutionOrder() >= BasePlugin.DEFAULT_CATALOGCALCULATION:
161  plug = PluginClass(config, name, self.schema, metadata=self.plugMetadata)
162  self.plugins[name] = plug
163  if plug.plugType == 'single':
164  self.executionDict[executionOrder].single.append(plug)
165  elif plug.plugType == 'multi':
166  self.executionDict[executionOrder].multi.append(plug)
167  else:
168  errorTuple = (PluginClass, PluginClass.getExecutionOrder(),
169  BasePlugin.DEFAULT_CATALOGCALCULATION)
170  raise ValueError("{} has an execution order less than the minimum for an catalogCalculation "
171  "plugin. Value {} : Minimum {}".format(*errorTuple))
172 
173  @lsst.pipe.base.timeMethod
174  def run(self, measCat):
175  '''
176  The entry point for the catalogCalculation task. This method should be called with a reference to a
177  measurement catalog.
178  '''
179  self.callCompute(measCat)
180 
181  def callCompute(self, catalog):
182  '''
183  Run each of the plugins on the catalog
184  @param[in] catalog The catalog on which the plugins will operate
185  '''
186  for runlevel in sorted(self.executionDict):
187  # Run all of the plugins which take a whole catalog first
188  for plug in self.executionDict[runlevel].multi:
189  with CCContext(plug, catalog, self.log):
190  plug.calculate(catalog)
191  # Run all the plugins which take single catalog entries
192  for measRecord in catalog:
193  for plug in self.executionDict[runlevel].single:
194  with CCContext(plug, measRecord, self.log):
195  plug.calculate(measRecord)
def __exit__(self, exc_type, exc_value, traceback)
Base class for measurement plugins.
Definition: pluginsBase.py:42
def __init__(self, schema, plugMetadata=None, kwargs)
Base class measurement Plugin config classes.
Definition: pluginsBase.py:33
Base class for plugin registries.
Map of plugins (instances of subclasses of BasePlugin) to be run for a task.
def __init__(self, config, name, schema, metadata)
Initialize the catalogCalculation plugin.
def calculate(self, cat, kwargs)
Process either a single catalog enter or the whole catalog and produce output defined by the plugin...