Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

from __future__ import print_function 

from builtins import zip 

from builtins import object 

import inspect 

import warnings 

import numpy as np 

from future.utils import with_metaclass 

 

__all__ = ['StackerRegistry', 'BaseStacker'] 

 

 

class StackerRegistry(type): 

""" 

Meta class for Stackers, to build a registry of stacker classes. 

""" 

 

def __init__(cls, name, bases, dict): 

super(StackerRegistry, cls).__init__(name, bases, dict) 

if not hasattr(cls, 'registry'): 

cls.registry = {} 

if not hasattr(cls, 'sourceDict'): 

cls.sourceDict = {} 

modname = inspect.getmodule(cls).__name__ 

24 ↛ 27line 24 didn't jump to line 27, because the condition on line 24 was never false if modname.startswith('lsst.sims.maf.stackers'): 

modname = '' 

else: 

if len(modname.split('.')) > 1: 

modname = '.'.join(modname.split('.')[:-1]) + '.' 

else: 

modname = modname + '.' 

stackername = modname + name 

32 ↛ 33line 32 didn't jump to line 33, because the condition on line 32 was never true if stackername in cls.registry: 

raise Exception('Redefining stacker %s! (there are >1 stackers with the same name)' 

% (stackername)) 

if stackername != 'BaseStacker': 

cls.registry[stackername] = cls 

colsAdded = cls.colsAdded 

for col in colsAdded: 

cls.sourceDict[col] = cls 

 

def getClass(cls, stackername): 

return cls.registry[stackername] 

 

def help(cls, doc=False): 

for stackername in sorted(cls.registry): 

if not doc: 

print(stackername) 

if doc: 

print('---- ', stackername, ' ----') 

print(cls.registry[stackername].__doc__) 

stacker = cls.registry[stackername]() 

print(' Columns added to SimData: ', ','.join(stacker.colsAdded)) 

print(' Default columns required: ', ','.join(stacker.colsReq)) 

 

 

class BaseStacker(with_metaclass(StackerRegistry, object)): 

"""Base MAF Stacker: add columns generated at run-time to the simdata array.""" 

# List of the names of the columns generated by the Stacker. 

colsAdded = [] 

 

def __init__(self): 

""" 

Instantiate the stacker. 

This method should be overriden by the user. This serves as an example of 

the variables required by the framework. 

""" 

# Add the list of new columns generated by the stacker as class attributes (colsAdded - above). 

# List of the names of the columns required from the database (to generate the Stacker columns). 

self.colsReq = [None] 

# Optional: specify the new column types. 

self.colsAddedDtypes = None 

# Optional: provide a list of units for the columns defined in colsAdded. 

self.units = [None] 

 

def __hash__(self): 

return None 

 

def __eq__(self, otherStacker): 

""" 

Evaluate if two stackers are equivalent. 

""" 

# If the class names are different, they are not 'the same'. 

if self.__class__.__name__ != otherStacker.__class__.__name__: 

return False 

# Otherwise, this is the same stacker class, but may be instantiated differently. 

# We have to delve a little further, and compare the kwargs & attributes for each stacker. 

stateNow = dir(self) 

for key in stateNow: 

if not key.startswith('_') and key != 'registry' and key != 'run' and key != 'next': 

if not hasattr(otherStacker, key): 

return False 

# If the attribute is from numpy, assume it's an array and test it 

if type(getattr(self, key)).__module__ == np.__name__: 

if not np.array_equal(getattr(self, key), getattr(otherStacker, key)): 

return False 

else: 

97 ↛ 98line 97 didn't jump to line 98, because the condition on line 97 was never true if getattr(self, key) != getattr(otherStacker, key): 

return False 

return True 

 

def __ne__(self, otherStacker): 

""" 

Evaluate if two stackers are not equal. 

""" 

105 ↛ 106line 105 didn't jump to line 106, because the condition on line 105 was never true if self == otherStacker: 

return False 

else: 

return True 

 

def _addStackerCols(self, simData): 

""" 

Add the new Stacker columns to the simData array. 

If columns already present in simData, just allows 'run' method to overwrite. 

Returns simData array with these columns added (so 'run' method can set their values). 

""" 

if not hasattr(self, 'colsAddedDtypes') or self.colsAddedDtypes is None: 

self.colsAddedDtypes = [float for col in self.colsAdded] 

# Create description of new recarray. 

newdtype = simData.dtype.descr 

cols_present = [False] * len(self.colsAdded) 

for i, (col, dtype) in enumerate(zip(self.colsAdded, self.colsAddedDtypes)): 

if col in simData.dtype.names: 

123 ↛ 121line 123 didn't jump to line 121, because the condition on line 123 was never false if simData[col][0] is not None: 

cols_present[i] = True 

warnings.warn('Warning - column %s already present in simData, may be overwritten ' 

'(depending on stacker).' 

% (col)) 

else: 

newdtype += ([(col, dtype)]) 

newData = np.empty(simData.shape, dtype=newdtype) 

# Add references to old data. 

for col in simData.dtype.names: 

newData[col] = simData[col] 

# Were all columns present and populated with something not None? If so, then consider 'all there'. 

if sum(cols_present) == len(self.colsAdded): 

cols_present = True 

else: 

cols_present = False 

return newData, cols_present 

 

def run(self, simData, override=False): 

""" 

Example: Generate the new stacker columns, given the simdata columns from the database. 

Returns the new simdata structured array that includes the new stacker columns. 

""" 

# Add new columns 

147 ↛ 148line 147 didn't jump to line 148, because the condition on line 147 was never true if len(simData) == 0: 

return simData 

simData, cols_present = self._addStackerCols(simData) 

# If override is set, it means go ahead and recalculate stacker values. 

if override: 

cols_present = False 

# Run the method to calculate/add new data. 

try: 

return self._run(simData, cols_present) 

except TypeError: 

warnings.warn('Please update the stacker %s so that the _run method matches the current API. ' 

'This will give you the option to skip re-running stackers if the columns are ' 

'already present.' 

% (self.__class__.__name__)) 

return self._run(simData) 

 

def _run(self, simData, cols_present=False): 

# By moving the calculation of these columns to a separate method, we add the possibility of using 

# stackers with pandas dataframes. The _addStackerCols method won't work with dataframes, but the 

# _run methods are quite likely to (depending on their details), as they are just populating columns. 

raise NotImplementedError('Not Implemented: ' 

'the child stackers should implement their own _run methods')