lsst.meas.base g5fe48ebfad+048d65dfd0
catalogCalculation.py
Go to the documentation of this file.
1# This file is part of meas_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
21
22from collections import namedtuple
23
24import lsst.pipe.base
25import lsst.pex.config
26import lsst.daf.base
27
28from .pluginsBase import BasePlugin, BasePluginConfig
29from .pluginRegistry import PluginRegistry, PluginMap
30from . import FatalAlgorithmError, MeasurementError
31
32# Exceptions that the measurement tasks should always propagate up to their
33# callers
34FATAL_EXCEPTIONS = (MemoryError, FatalAlgorithmError)
35
36__all__ = ("CatalogCalculationPluginConfig", "CatalogCalculationPlugin", "CatalogCalculationConfig",
37 "CatalogCalculationTask")
38
39
41 """Default configuration class for catalog calcuation plugins.
42 """
43 pass
44
45
47 """Base class for catalog calculation plugins.
48
49 Parameters
50 ----------
51 config : `CatalogCalculationPlugin.ConfigClass`
52 Plugin configuration.
53 name : `str`
54 The string the plugin was registered with.
55 schema : `lsst.afw.table.Schema`
56 The source schema, New fields should be added here to
57 hold output produced by this plugin.
58 metadata : `lsst.daf.base.PropertySet`
59 Plugin metadata that will be attached to the output catalog
60 """
61
62 ConfigClass = CatalogCalculationPluginConfig # documentation inherited
63
64 registry = PluginRegistry(CatalogCalculationPluginConfig)
65 """List of available plugins (`lsst.meas.base.PluginRegistry`).
66 """
67
68 plugType = 'single'
69 """Does the plugin operate on a single source or the whole catalog (`str`)?
70
71 If the plugin operates on a single source at a time, this should be set to
72 ``"single"``; if it expects the whoe catalog, to ``"multi"``. If the
73 plugin is of type ``"multi"``, the `fail` method must be implemented to
74 accept the whole catalog. If the plugin is of type ``"single"``, `fail`
75 should accept a single source record.
76 """
77
78 def __init__(self, config, name, schema, metadata):
79 BasePlugin.__init__(self, config, name)
80
81 @classmethod
83 r"""Used to set the relative order of plugin execution.
84
85 The values returned by `getExecutionOrder` are compared across all
86 plugins, and smaller numbers run first.
87
88 Notes
89 -----
90 `CatalogCalculationPlugin`\s must run with
91 `BasePlugin.DEFAULT_CATALOGCALCULATION` or higher.
92
93 All plugins must implement this method with an appropriate run level
94 """
95 raise NotImplementedError()
96
97 def calculate(self, cat, **kwargs):
98 """Perform the calculation specified by this plugin.
99
100 This method can either be used to operate on a single catalog record
101 or a whole catalog, populating it with the output defined by this
102 plugin.
103
104 Note that results may be added to catalog records as new columns, or
105 may result in changes to existing values.
106
107 Parameters
108 ----------
110 May either be a `~lsst.afw.table.SourceCatalog` or a single
111 `~lsst.afw.table.SourceRecord`, depending on the plugin type. Will
112 be updated in place to contain the results of plugin execution.
113 **kwargs
114 Any additional keyword arguments that may be passed to the plugin.
115 """
116 raise NotImplementedError()
117
118
120 """Handle errors that are thrown by catalog calculation plugins.
121
122 This is a context manager.
123
124 Parameters
125 ----------
126 plugin : `CatalogCalculationPlugin`
127 The plugin that is to be run.
129 May either be a `~lsst.afw.table.SourceCatalog` or a single
130 `~lsst.afw.table.SourceRecord`, depending on the plugin type.
131 log : `lsst.log.Log` or `logging.Logger`
132 A logger. Generally, this should be the logger of the object in which
133 the context manager is being used.
134 """
135 def __init__(self, plugin, cat, log):
136 self.pluginplugin = plugin
137 self.catcat = cat
138 self.loglog = log
139
140 def __enter__(self):
141 return
142
143 def __exit__(self, exc_type, exc_value, traceback):
144 if exc_type is None:
145 return True
146 if exc_type in FATAL_EXCEPTIONS:
147 raise exc_value
148 elif exc_type is MeasurementError:
149 self.plugin.fail(self.cat, exc_value)
150 else:
151 self.log.warning("Error in %s.calculate: %s", self.plugin.name, exc_value)
152 return True
153
154
155class CatalogCalculationConfig(lsst.pex.config.Config):
156 """Config class for the catalog calculation driver task.
157
158 Specifies which plugins will execute when the `CatalogCalculationTask`
159 associated with this configuration is run.
160 """
161
162 plugins = CatalogCalculationPlugin.registry.makeField(
163 multi=True,
164 default=["base_ClassificationExtendedness",
165 "base_FootprintArea"],
166 doc="Plugins to be run and their configuration")
167
168
169class CatalogCalculationTask(lsst.pipe.base.Task):
170 """Run plugins which operate on a catalog of sources.
171
172 This task facilitates running plugins which will operate on a source
173 catalog. These plugins may do things such as classifying an object based
174 on source record entries inserted during a measurement task.
175
176 Parameters
177 ----------
178 plugMetaData : `lsst.daf.base.PropertyList` or `None`
179 Will be modified in-place to contain metadata about the plugins being
180 run. If `None`, an empty `~lsst.daf.base.PropertyList` will be
181 created.
182 **kwargs
183 Additional arguments passed to the superclass constructor.
184
185 Notes
186 -----
187 Plugins may either take an entire catalog to work on at a time, or work on
188 individual records.
189 """
190 ConfigClass = CatalogCalculationConfig
191 _DefaultName = "catalogCalculation"
192
193 def __init__(self, schema, plugMetadata=None, **kwargs):
194 lsst.pipe.base.Task.__init__(self, **kwargs)
195 self.schemaschema = schema
196 if plugMetadata is None:
197 plugMetadata = lsst.daf.base.PropertyList()
198 self.plugMetadataplugMetadata = plugMetadata
199 self.pluginsplugins = PluginMap()
200
201 self.initializePluginsinitializePlugins()
202
204 """Initialize the plugins according to the configuration.
205 """
206
207 pluginType = namedtuple('pluginType', 'single multi')
208 self.executionDictexecutionDict = {}
209 # Read the properties for each plugin. Allocate a dictionary entry for each run level. Verify that
210 # the plugins are above the minimum run level for an catalogCalculation plugin. For each run level,
211 # the plugins are sorted into either single record, or multi record groups to later be run
212 # appropriately
213 for executionOrder, name, config, PluginClass in sorted(self.config.plugins.apply()):
214 if executionOrder not in self.executionDictexecutionDict:
215 self.executionDictexecutionDict[executionOrder] = pluginType(single=[], multi=[])
216 if PluginClass.getExecutionOrder() >= BasePlugin.DEFAULT_CATALOGCALCULATION:
217 plug = PluginClass(config, name, self.schemaschema, metadata=self.plugMetadataplugMetadata)
218 self.pluginsplugins[name] = plug
219 if plug.plugType == 'single':
220 self.executionDictexecutionDict[executionOrder].single.append(plug)
221 elif plug.plugType == 'multi':
222 self.executionDictexecutionDict[executionOrder].multi.append(plug)
223 else:
224 errorTuple = (PluginClass, PluginClass.getExecutionOrder(),
225 BasePlugin.DEFAULT_CATALOGCALCULATION)
226 raise ValueError("{} has an execution order less than the minimum for an catalogCalculation "
227 "plugin. Value {} : Minimum {}".format(*errorTuple))
228
229 @lsst.pipe.base.timeMethod
230 def run(self, measCat):
231 """The entry point for the catalog calculation task.
232
233 Parameters
234 ----------
236 Catalog for measurement.
237 """
238 self.callComputecallCompute(measCat)
239
240 def callCompute(self, catalog):
241 """Run each of the plugins on the catalog.
242
243 Parameters
244 ----------
246 The catalog on which the plugins will operate.
247 """
248 for runlevel in sorted(self.executionDictexecutionDict):
249 # Run all of the plugins which take a whole catalog first
250 for plug in self.executionDictexecutionDict[runlevel].multi:
251 with CCContext(plug, catalog, self.log):
252 plug.calculate(catalog)
253 # Run all the plugins which take single catalog entries
254 for measRecord in catalog:
255 for plug in self.executionDictexecutionDict[runlevel].single:
256 with CCContext(plug, measRecord, self.log):
257 plug.calculate(measRecord)
def __exit__(self, exc_type, exc_value, traceback)
def __init__(self, config, name, schema, metadata)
def __init__(self, schema, plugMetadata=None, **kwargs)