Coverage for python/lsst/meas/base/catalogCalculation.py: 38%
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
Shortcuts on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# This file is part of meas_base.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (https://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <https://www.gnu.org/licenses/>.
22from collections import namedtuple
24import lsst.pipe.base
25import lsst.pex.config
26import lsst.daf.base
28from .pluginsBase import BasePlugin, BasePluginConfig
29from .pluginRegistry import PluginRegistry, PluginMap
30from . import FatalAlgorithmError, MeasurementError
32# Exceptions that the measurement tasks should always propagate up to their
33# callers
34FATAL_EXCEPTIONS = (MemoryError, FatalAlgorithmError)
36__all__ = ("CatalogCalculationPluginConfig", "CatalogCalculationPlugin", "CatalogCalculationConfig",
37 "CatalogCalculationTask")
40class CatalogCalculationPluginConfig(BasePluginConfig):
41 """Default configuration class for catalog calcuation plugins.
42 """
43 pass
46class CatalogCalculationPlugin(BasePlugin):
47 """Base class for catalog calculation plugins.
49 Parameters
50 ----------
51 config : `CatalogCalculationPlugin.ConfigClass`
52 Plugin configuration.
53 name : `str`
54 The string the plugin was registered with.
55 schema : `lsst.afw.table.Schema`
56 The source schema, New fields should be added here to
57 hold output produced by this plugin.
58 metadata : `lsst.daf.base.PropertySet`
59 Plugin metadata that will be attached to the output catalog
60 """
62 ConfigClass = CatalogCalculationPluginConfig # documentation inherited
64 registry = PluginRegistry(CatalogCalculationPluginConfig)
65 """List of available plugins (`lsst.meas.base.PluginRegistry`).
66 """
68 plugType = 'single'
69 """Does the plugin operate on a single source or the whole catalog (`str`)?
71 If the plugin operates on a single source at a time, this should be set to
72 ``"single"``; if it expects the whoe catalog, to ``"multi"``. If the
73 plugin is of type ``"multi"``, the `fail` method must be implemented to
74 accept the whole catalog. If the plugin is of type ``"single"``, `fail`
75 should accept a single source record.
76 """
78 def __init__(self, config, name, schema, metadata):
79 BasePlugin.__init__(self, config, name)
81 @classmethod
82 def getExecutionOrder(cls):
83 r"""Used to set the relative order of plugin execution.
85 The values returned by `getExecutionOrder` are compared across all
86 plugins, and smaller numbers run first.
88 Notes
89 -----
90 `CatalogCalculationPlugin`\s must run with
91 `BasePlugin.DEFAULT_CATALOGCALCULATION` or higher.
93 All plugins must implement this method with an appropriate run level
94 """
95 raise NotImplementedError()
97 def calculate(self, cat, **kwargs):
98 """Perform the calculation specified by this plugin.
100 This method can either be used to operate on a single catalog record
101 or a whole catalog, populating it with the output defined by this
102 plugin.
104 Note that results may be added to catalog records as new columns, or
105 may result in changes to existing values.
107 Parameters
108 ----------
109 cat : `lsst.afw.table.SourceCatalog` or `lsst.afw.table.SourceRecord`
110 May either be a `~lsst.afw.table.SourceCatalog` or a single
111 `~lsst.afw.table.SourceRecord`, depending on the plugin type. Will
112 be updated in place to contain the results of plugin execution.
113 **kwargs
114 Any additional keyword arguments that may be passed to the plugin.
115 """
116 raise NotImplementedError()
119class CCContext:
120 """Handle errors that are thrown by catalog calculation plugins.
122 This is a context manager.
124 Parameters
125 ----------
126 plugin : `CatalogCalculationPlugin`
127 The plugin that is to be run.
128 cat : `lsst.afw.table.SourceCatalog` or `lsst.afw.table.SourceRecord`
129 May either be a `~lsst.afw.table.SourceCatalog` or a single
130 `~lsst.afw.table.SourceRecord`, depending on the plugin type.
131 log : `lsst.log.Log` or `logging.Logger`
132 A logger. Generally, this should be the logger of the object in which
133 the context manager is being used.
134 """
135 def __init__(self, plugin, cat, log):
136 self.plugin = plugin
137 self.cat = cat
138 self.log = log
140 def __enter__(self):
141 return
143 def __exit__(self, exc_type, exc_value, traceback):
144 if exc_type is None:
145 return True
146 if exc_type in FATAL_EXCEPTIONS:
147 raise exc_value
148 elif exc_type is MeasurementError:
149 self.plugin.fail(self.cat, exc_value)
150 else:
151 self.log.warning("Error in %s.calculate: %s", self.plugin.name, exc_value)
152 return True
155class CatalogCalculationConfig(lsst.pex.config.Config):
156 """Config class for the catalog calculation driver task.
158 Specifies which plugins will execute when the `CatalogCalculationTask`
159 associated with this configuration is run.
160 """
162 plugins = CatalogCalculationPlugin.registry.makeField(
163 multi=True,
164 default=["base_ClassificationExtendedness",
165 "base_FootprintArea"],
166 doc="Plugins to be run and their configuration")
169class CatalogCalculationTask(lsst.pipe.base.Task):
170 """Run plugins which operate on a catalog of sources.
172 This task facilitates running plugins which will operate on a source
173 catalog. These plugins may do things such as classifying an object based
174 on source record entries inserted during a measurement task.
176 Parameters
177 ----------
178 plugMetaData : `lsst.daf.base.PropertyList` or `None`
179 Will be modified in-place to contain metadata about the plugins being
180 run. If `None`, an empty `~lsst.daf.base.PropertyList` will be
181 created.
182 **kwargs
183 Additional arguments passed to the superclass constructor.
185 Notes
186 -----
187 Plugins may either take an entire catalog to work on at a time, or work on
188 individual records.
189 """
190 ConfigClass = CatalogCalculationConfig
191 _DefaultName = "catalogCalculation"
193 def __init__(self, schema, plugMetadata=None, **kwargs):
194 lsst.pipe.base.Task.__init__(self, **kwargs)
195 self.schema = schema
196 if plugMetadata is None:
197 plugMetadata = lsst.daf.base.PropertyList()
198 self.plugMetadata = plugMetadata
199 self.plugins = PluginMap()
201 self.initializePlugins()
203 def initializePlugins(self):
204 """Initialize the plugins according to the configuration.
205 """
207 pluginType = namedtuple('pluginType', 'single multi')
208 self.executionDict = {}
209 # Read the properties for each plugin. Allocate a dictionary entry for each run level. Verify that
210 # the plugins are above the minimum run level for an catalogCalculation plugin. For each run level,
211 # the plugins are sorted into either single record, or multi record groups to later be run
212 # appropriately
213 for executionOrder, name, config, PluginClass in sorted(self.config.plugins.apply()):
214 if executionOrder not in self.executionDict:
215 self.executionDict[executionOrder] = pluginType(single=[], multi=[])
216 if PluginClass.getExecutionOrder() >= BasePlugin.DEFAULT_CATALOGCALCULATION:
217 plug = PluginClass(config, name, self.schema, metadata=self.plugMetadata)
218 self.plugins[name] = plug
219 if plug.plugType == 'single':
220 self.executionDict[executionOrder].single.append(plug)
221 elif plug.plugType == 'multi':
222 self.executionDict[executionOrder].multi.append(plug)
223 else:
224 errorTuple = (PluginClass, PluginClass.getExecutionOrder(),
225 BasePlugin.DEFAULT_CATALOGCALCULATION)
226 raise ValueError("{} has an execution order less than the minimum for an catalogCalculation "
227 "plugin. Value {} : Minimum {}".format(*errorTuple))
229 @lsst.pipe.base.timeMethod
230 def run(self, measCat):
231 """The entry point for the catalog calculation task.
233 Parameters
234 ----------
235 meascat : `lsst.afw.table.SourceCatalog`
236 Catalog for measurement.
237 """
238 self.callCompute(measCat)
240 def callCompute(self, catalog):
241 """Run each of the plugins on the catalog.
243 Parameters
244 ----------
245 catalog : `lsst.afw.table.SourceCatalog`
246 The catalog on which the plugins will operate.
247 """
248 for runlevel in sorted(self.executionDict):
249 # Run all of the plugins which take a whole catalog first
250 for plug in self.executionDict[runlevel].multi:
251 with CCContext(plug, catalog, self.log):
252 plug.calculate(catalog)
253 # Run all the plugins which take single catalog entries
254 for measRecord in catalog:
255 for plug in self.executionDict[runlevel].single:
256 with CCContext(plug, measRecord, self.log):
257 plug.calculate(measRecord)