Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

from collections import namedtuple 

 

import lsst.pipe.base 

import lsst.pex.config 

import lsst.daf.base 

 

from .pluginsBase import BasePlugin, BasePluginConfig 

from .pluginRegistry import PluginRegistry, PluginMap 

from . import FatalAlgorithmError, MeasurementError 

 

# Exceptions that the measurement tasks should always propagate up to their callers 

FATAL_EXCEPTIONS = (MemoryError, FatalAlgorithmError) 

 

__all__ = ("CatalogCalculationPluginConfig", "CatalogCalculationPlugin", "CatalogCalculationConfig", 

"CatalogCalculationTask") 

 

 

class CatalogCalculationPluginConfig(BasePluginConfig): 

''' 

Default configuration class for catalogCalcuation plugins 

''' 

pass 

 

 

class CatalogCalculationPlugin(BasePlugin): 

''' 

Base class for after CatalogCalculation plugin 

''' 

registry = PluginRegistry(CatalogCalculationPluginConfig) 

ConfigClass = CatalogCalculationPluginConfig 

# This defines if the plugin operates on a single source at a time, or expects the whole catalog. 

# The value defaults to single for a single source, set to multi when the plugin expects the whole 

# catalog. If The plugin is of type multi, the fail method should be implemented to accept the whole 

# catalog. If the plugin is of type the fail method should accept a single source record. 

 

plugType = 'single' 

 

def __init__(self, config, name, schema, metadata): 

"""! 

Initialize the catalogCalculation plugin 

 

@param[in] config An instance of catalogCalculation config class. 

@param[in] name The string the plugin was registered with. 

@param[in,out] schema The source schema, New fields should be added here to 

hold output produced by this plugin. 

@param[in] metadata Plugin metadata that will be attached to the output catalog 

""" 

BasePlugin.__init__(self, config, name) 

 

@classmethod 

def getExecutionOrder(cls): 

''' Sets the relative order of plugins (smaller numbers run first). 

 

CatalogCalculation plugins must run with BasePlugin.DEFAULT_CATALOGCALCULATION or higher 

 

All plugins must implement this method with an appropriate run level 

''' 

raise NotImplementedError() 

 

def calculate(self, cat, **kwargs): 

"""! 

Process either a single catalog enter or the whole catalog and produce output defined by the plugin 

 

@param[in,out] cat Either a lsst source catalog or a catalog entery depending on the plug type 

specified in the classes configuration. Results may be added to new columns, 

or existing entries altered. 

@param[in] kwargs Any additional kwargs that may be passed through the CatalogCalculationPlugin. 

""" 

raise NotImplementedError() 

 

 

class CCContext: 

''' 

Context manager to handle catching errors that may have been thrown in a catalogCalculation plugin 

@param[in] plugin The plugin that is to be run 

@param[in] cat Either a catalog or a source record entry of a catalog, depending of the plugin type, 

i.e. either working on a whole catalog, or a single record. 

@param[in] log The log which to write to, most likely will always be the log (self.log) of the object 

in which the context manager is used. 

''' 

 

def __init__(self, plugin, cat, log): 

self.plugin = plugin 

self.cat = cat 

self.log = log 

 

def __enter__(self): 

return 

 

def __exit__(self, exc_type, exc_value, traceback): 

if exc_type is None: 

return True 

93 ↛ 94line 93 didn't jump to line 94, because the condition on line 93 was never true if exc_type in FATAL_EXCEPTIONS: 

raise exc_value 

95 ↛ 98line 95 didn't jump to line 98, because the condition on line 95 was never false elif exc_type is MeasurementError: 

self.plugin.fail(self.cat, exc_value) 

else: 

self.log.warn("Error in {}.calculate: {}".format(self.plugin.name, exc_value)) 

return True 

 

 

class CatalogCalculationConfig(lsst.pex.config.Config): 

''' 

Config class for catalog calculation driver task. 

 

Specifies which plugins will execute when CatalogCalculationTask 

associated with this configuration is run. 

''' 

plugins = CatalogCalculationPlugin.registry.makeField( 

multi=True, 

default=["base_ClassificationExtendedness", 

"base_FootprintArea"], 

doc="Plugins to be run and their configuration") 

 

 

class CatalogCalculationTask(lsst.pipe.base.Task): 

''' 

This task facilitates running plugins which will operate on a source catalog. These plugins may do things 

such as classifying an object based on source record entries inserted during a measurement task. 

 

Plugins may either take an entire catalog to work on at a time, or work on individual records 

''' 

ConfigClass = CatalogCalculationConfig 

_DefaultName = "catalogCalculation" 

 

def __init__(self, schema, plugMetadata=None, **kwargs): 

""" 

Constructor; only called by derived classes 

 

@param[in] plugMetaData An lsst.daf.base.PropertyList that will be filled with metadata 

about the plugins being run. If None, an empty empty PropertyList 

will be created. 

@param[in] **kwargs Additional arguments passed to lsst.pipe.base.Task.__init__. 

""" 

lsst.pipe.base.Task.__init__(self, **kwargs) 

self.schema = schema 

137 ↛ 139line 137 didn't jump to line 139, because the condition on line 137 was never false if plugMetadata is None: 

plugMetadata = lsst.daf.base.PropertyList() 

self.plugMetadata = plugMetadata 

self.plugins = PluginMap() 

 

self.initializePlugins() 

 

def initializePlugins(self): 

''' 

Initialize the plugins according to the configuration. 

''' 

 

pluginType = namedtuple('pluginType', 'single multi') 

self.executionDict = {} 

# Read the properties for each plugin. Allocate a dictionary entry for each run level. Verify that 

# the plugins are above the minimum run level for an catalogCalculation plugin. For each run level, 

# the plugins are sorted into either single record, or multi record groups to later be run 

# appropriately 

for executionOrder, name, config, PluginClass in sorted(self.config.plugins.apply()): 

if executionOrder not in self.executionDict: 

self.executionDict[executionOrder] = pluginType(single=[], multi=[]) 

158 ↛ 166line 158 didn't jump to line 166, because the condition on line 158 was never false if PluginClass.getExecutionOrder() >= BasePlugin.DEFAULT_CATALOGCALCULATION: 

plug = PluginClass(config, name, self.schema, metadata=self.plugMetadata) 

self.plugins[name] = plug 

if plug.plugType == 'single': 

self.executionDict[executionOrder].single.append(plug) 

163 ↛ 155line 163 didn't jump to line 155, because the condition on line 163 was never false elif plug.plugType == 'multi': 

self.executionDict[executionOrder].multi.append(plug) 

else: 

errorTuple = (PluginClass, PluginClass.getExecutionOrder(), 

BasePlugin.DEFAULT_CATALOGCALCULATION) 

raise ValueError("{} has an execution order less than the minimum for an catalogCalculation " 

"plugin. Value {} : Minimum {}".format(*errorTuple)) 

 

@lsst.pipe.base.timeMethod 

def run(self, measCat): 

''' 

The entry point for the catalogCalculation task. This method should be called with a reference to a 

measurement catalog. 

''' 

self.callCompute(measCat) 

 

def callCompute(self, catalog): 

''' 

Run each of the plugins on the catalog 

@param[in] catalog The catalog on which the plugins will operate 

''' 

for runlevel in sorted(self.executionDict): 

# Run all of the plugins which take a whole catalog first 

for plug in self.executionDict[runlevel].multi: 

with CCContext(plug, catalog, self.log): 

plug.calculate(catalog) 

# Run all the plugins which take single catalog entries 

for measRecord in catalog: 

for plug in self.executionDict[runlevel].single: 

with CCContext(plug, measRecord, self.log): 

plug.calculate(measRecord)