Coverage for python/lsst/meas/base/catalogCalculation.py: 32%

78 statements  

« prev     ^ index     » next       coverage.py v7.1.0, created at 2023-02-05 18:07 -0800

1# This file is part of meas_base. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22from collections import namedtuple 

23 

24import lsst.pipe.base 

25import lsst.pex.config 

26import lsst.daf.base 

27 

28from .pluginsBase import BasePlugin, BasePluginConfig 

29from .pluginRegistry import PluginRegistry, PluginMap 

30from . import FatalAlgorithmError, MeasurementError 

31 

32# Exceptions that the measurement tasks should always propagate up to their 

33# callers 

34FATAL_EXCEPTIONS = (MemoryError, FatalAlgorithmError) 

35 

36__all__ = ("CatalogCalculationPluginConfig", "CatalogCalculationPlugin", "CatalogCalculationConfig", 

37 "CatalogCalculationTask") 

38 

39 

40class CatalogCalculationPluginConfig(BasePluginConfig): 

41 """Default configuration class for catalog calcuation plugins. 

42 """ 

43 pass 

44 

45 

46class CatalogCalculationPlugin(BasePlugin): 

47 """Base class for catalog calculation plugins. 

48 

49 Parameters 

50 ---------- 

51 config : `CatalogCalculationPlugin.ConfigClass` 

52 Plugin configuration. 

53 name : `str` 

54 The string the plugin was registered with. 

55 schema : `lsst.afw.table.Schema` 

56 The source schema, New fields should be added here to 

57 hold output produced by this plugin. 

58 metadata : `lsst.daf.base.PropertySet` 

59 Plugin metadata that will be attached to the output catalog 

60 """ 

61 

62 ConfigClass = CatalogCalculationPluginConfig # documentation inherited 

63 

64 registry = PluginRegistry(CatalogCalculationPluginConfig) 

65 """List of available plugins (`lsst.meas.base.PluginRegistry`). 

66 """ 

67 

68 plugType = 'single' 

69 """Does the plugin operate on a single source or the whole catalog (`str`)? 

70 

71 If the plugin operates on a single source at a time, this should be set to 

72 ``"single"``; if it expects the whoe catalog, to ``"multi"``. If the 

73 plugin is of type ``"multi"``, the `fail` method must be implemented to 

74 accept the whole catalog. If the plugin is of type ``"single"``, `fail` 

75 should accept a single source record. 

76 """ 

77 

78 def __init__(self, config, name, schema, metadata): 

79 BasePlugin.__init__(self, config, name) 

80 

81 @classmethod 

82 def getExecutionOrder(cls): 

83 r"""Used to set the relative order of plugin execution. 

84 

85 The values returned by `getExecutionOrder` are compared across all 

86 plugins, and smaller numbers run first. 

87 

88 Notes 

89 ----- 

90 `CatalogCalculationPlugin`\s must run with 

91 `BasePlugin.DEFAULT_CATALOGCALCULATION` or higher. 

92 

93 All plugins must implement this method with an appropriate run level 

94 """ 

95 raise NotImplementedError() 

96 

97 def calculate(self, cat, **kwargs): 

98 """Perform the calculation specified by this plugin. 

99 

100 This method can either be used to operate on a single catalog record 

101 or a whole catalog, populating it with the output defined by this 

102 plugin. 

103 

104 Note that results may be added to catalog records as new columns, or 

105 may result in changes to existing values. 

106 

107 Parameters 

108 ---------- 

109 cat : `lsst.afw.table.SourceCatalog` or `lsst.afw.table.SourceRecord` 

110 May either be a `~lsst.afw.table.SourceCatalog` or a single 

111 `~lsst.afw.table.SourceRecord`, depending on the plugin type. Will 

112 be updated in place to contain the results of plugin execution. 

113 **kwargs 

114 Any additional keyword arguments that may be passed to the plugin. 

115 """ 

116 raise NotImplementedError() 

117 

118 

119class CCContext: 

120 """Handle errors that are thrown by catalog calculation plugins. 

121 

122 This is a context manager. 

123 

124 Parameters 

125 ---------- 

126 plugin : `CatalogCalculationPlugin` 

127 The plugin that is to be run. 

128 cat : `lsst.afw.table.SourceCatalog` or `lsst.afw.table.SourceRecord` 

129 May either be a `~lsst.afw.table.SourceCatalog` or a single 

130 `~lsst.afw.table.SourceRecord`, depending on the plugin type. 

131 log : `lsst.log.Log` or `logging.Logger` 

132 A logger. Generally, this should be the logger of the object in which 

133 the context manager is being used. 

134 """ 

135 def __init__(self, plugin, cat, log): 

136 self.plugin = plugin 

137 self.cat = cat 

138 self.log = log 

139 

140 def __enter__(self): 

141 return 

142 

143 def __exit__(self, exc_type, exc_value, traceback): 

144 if exc_type is None: 

145 return True 

146 if exc_type in FATAL_EXCEPTIONS: 

147 raise exc_value 

148 elif exc_type is MeasurementError: 

149 self.plugin.fail(self.cat, exc_value) 

150 else: 

151 self.log.warning("Error in %s.calculate: %s", self.plugin.name, exc_value) 

152 return True 

153 

154 

155class CatalogCalculationConfig(lsst.pex.config.Config): 

156 """Config class for the catalog calculation driver task. 

157 

158 Specifies which plugins will execute when the `CatalogCalculationTask` 

159 associated with this configuration is run. 

160 """ 

161 

162 plugins = CatalogCalculationPlugin.registry.makeField( 

163 multi=True, 

164 default=["base_ClassificationExtendedness", 

165 "base_FootprintArea"], 

166 doc="Plugins to be run and their configuration") 

167 

168 

169class CatalogCalculationTask(lsst.pipe.base.Task): 

170 """Run plugins which operate on a catalog of sources. 

171 

172 This task facilitates running plugins which will operate on a source 

173 catalog. These plugins may do things such as classifying an object based 

174 on source record entries inserted during a measurement task. 

175 

176 Parameters 

177 ---------- 

178 plugMetaData : `lsst.daf.base.PropertyList` or `None` 

179 Will be modified in-place to contain metadata about the plugins being 

180 run. If `None`, an empty `~lsst.daf.base.PropertyList` will be 

181 created. 

182 **kwargs 

183 Additional arguments passed to the superclass constructor. 

184 

185 Notes 

186 ----- 

187 Plugins may either take an entire catalog to work on at a time, or work on 

188 individual records. 

189 """ 

190 ConfigClass = CatalogCalculationConfig 

191 _DefaultName = "catalogCalculation" 

192 

193 def __init__(self, schema, plugMetadata=None, **kwargs): 

194 lsst.pipe.base.Task.__init__(self, **kwargs) 

195 self.schema = schema 

196 if plugMetadata is None: 

197 plugMetadata = lsst.daf.base.PropertyList() 

198 self.plugMetadata = plugMetadata 

199 self.plugins = PluginMap() 

200 

201 self.initializePlugins() 

202 

203 def initializePlugins(self): 

204 """Initialize the plugins according to the configuration. 

205 """ 

206 

207 pluginType = namedtuple('pluginType', 'single multi') 

208 self.executionDict = {} 

209 # Read the properties for each plugin. Allocate a dictionary entry for each run level. Verify that 

210 # the plugins are above the minimum run level for an catalogCalculation plugin. For each run level, 

211 # the plugins are sorted into either single record, or multi record groups to later be run 

212 # appropriately 

213 for executionOrder, name, config, PluginClass in sorted(self.config.plugins.apply()): 

214 if executionOrder not in self.executionDict: 

215 self.executionDict[executionOrder] = pluginType(single=[], multi=[]) 

216 if PluginClass.getExecutionOrder() >= BasePlugin.DEFAULT_CATALOGCALCULATION: 

217 plug = PluginClass(config, name, self.schema, metadata=self.plugMetadata) 

218 self.plugins[name] = plug 

219 if plug.plugType == 'single': 

220 self.executionDict[executionOrder].single.append(plug) 

221 elif plug.plugType == 'multi': 

222 self.executionDict[executionOrder].multi.append(plug) 

223 else: 

224 errorTuple = (PluginClass, PluginClass.getExecutionOrder(), 

225 BasePlugin.DEFAULT_CATALOGCALCULATION) 

226 raise ValueError("{} has an execution order less than the minimum for an catalogCalculation " 

227 "plugin. Value {} : Minimum {}".format(*errorTuple)) 

228 

229 @lsst.pipe.base.timeMethod 

230 def run(self, measCat): 

231 """The entry point for the catalog calculation task. 

232 

233 Parameters 

234 ---------- 

235 meascat : `lsst.afw.table.SourceCatalog` 

236 Catalog for measurement. 

237 """ 

238 self.callCompute(measCat) 

239 

240 def callCompute(self, catalog): 

241 """Run each of the plugins on the catalog. 

242 

243 Parameters 

244 ---------- 

245 catalog : `lsst.afw.table.SourceCatalog` 

246 The catalog on which the plugins will operate. 

247 """ 

248 for runlevel in sorted(self.executionDict): 

249 # Run all of the plugins which take a whole catalog first 

250 for plug in self.executionDict[runlevel].multi: 

251 with CCContext(plug, catalog, self.log): 

252 plug.calculate(catalog) 

253 # Run all the plugins which take single catalog entries 

254 for measRecord in catalog: 

255 for plug in self.executionDict[runlevel].single: 

256 with CCContext(plug, measRecord, self.log): 

257 plug.calculate(measRecord)