Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

from __future__ import print_function 

from builtins import zip 

from builtins import object 

import inspect 

import warnings 

import numpy as np 

from future.utils import with_metaclass 

 

__all__ = ['StackerRegistry', 'BaseStacker'] 

 

 

class StackerRegistry(type): 

""" 

Meta class for Stackers, to build a registry of stacker classes. 

""" 

def __init__(cls, name, bases, dict): 

super(StackerRegistry, cls).__init__(name, bases, dict) 

if not hasattr(cls, 'registry'): 

cls.registry = {} 

modname = inspect.getmodule(cls).__name__ 

21 ↛ 24line 21 didn't jump to line 24, because the condition on line 21 was never false if modname.startswith('lsst.sims.maf.stackers'): 

modname = '' 

else: 

if len(modname.split('.')) > 1: 

modname = '.'.join(modname.split('.')[:-1]) + '.' 

else: 

modname = modname + '.' 

stackername = modname + name 

29 ↛ 30line 29 didn't jump to line 30, because the condition on line 29 was never true if stackername in cls.registry: 

raise Exception('Redefining stacker %s! (there are >1 stackers with the same name)' 

% (stackername)) 

if stackername != 'BaseStacker': 

cls.registry[stackername] = cls 

 

def getClass(cls, stackername): 

return cls.registry[stackername] 

 

def help(cls, doc=False): 

for stackername in sorted(cls.registry): 

if not doc: 

print(stackername) 

if doc: 

print('---- ', stackername, ' ----') 

print(cls.registry[stackername].__doc__) 

stacker = cls.registry[stackername]() 

print(' Columns added to SimData: ', ','.join(stacker.colsAdded)) 

print(' Default columns required: ', ','.join(stacker.colsReq)) 

 

 

class BaseStacker(with_metaclass(StackerRegistry, object)): 

"""Base MAF Stacker: add columns generated at run-time to the simdata array.""" 

 

def __init__(self): 

""" 

Instantiate the stacker. 

This method should be overriden by the user. This serves as an example of 

the variables required by the framework. 

""" 

# List of the names of the columns generated by the Stacker. 

self.colsAdded = [None] 

# List of the names of the columns required from the database (to generate the Stacker columns). 

self.colsReq = [None] 

# Optional: specify the new column types. 

self.colsAddedDtypes = None 

# Optional: provide a list of units for the columns defined in colsAdded. 

self.units = [None] 

 

def __hash__(self): 

return None 

 

def __eq__(self, otherStacker): 

""" 

Evaluate if two stackers are equivalent. 

""" 

# If the class names are different, they are not 'the same'. 

if self.__class__.__name__ != otherStacker.__class__.__name__: 

return False 

# Otherwise, this is the same stacker class, but may be instantiated differently. 

# We have to delve a little further, and compare the kwargs & attributes for each stacker. 

stateNow = dir(self) 

for key in stateNow: 

if not key.startswith('_') and key != 'registry' and key != 'run' and key != 'next': 

if not hasattr(otherStacker, key): 

return False 

# If the attribute is from numpy, assume it's an array and test it 

if type(getattr(self, key)).__module__ == np.__name__: 

if not np.array_equal(getattr(self, key), getattr(otherStacker, key)): 

return False 

else: 

90 ↛ 91line 90 didn't jump to line 91, because the condition on line 90 was never true if getattr(self, key) != getattr(otherStacker, key): 

return False 

return True 

 

def __ne__(self, otherStacker): 

""" 

Evaluate if two stackers are not equal. 

""" 

98 ↛ 99line 98 didn't jump to line 99, because the condition on line 98 was never true if self == otherStacker: 

return False 

else: 

return True 

 

def _addStackerCols(self, simData): 

""" 

Add the new Stacker columns to the simData array. 

If columns already present in simData, just allows 'run' method to overwrite. 

Returns simData array with these columns added (so 'run' method can set their values). 

""" 

if not hasattr(self, 'colsAddedDtypes') or self.colsAddedDtypes is None: 

self.colsAddedDtypes = [float for col in self.colsAdded] 

# Create description of new recarray. 

newdtype = simData.dtype.descr 

cols_present = [False] * len(self.colsAdded) 

for i, (col, dtype) in enumerate(zip(self.colsAdded, self.colsAddedDtypes)): 

if col in simData.dtype.names: 

116 ↛ 114line 116 didn't jump to line 114, because the condition on line 116 was never false if simData[col][0] is not None: 

cols_present[i] = True 

warnings.warn('Warning - column %s already present in simData, may be overwritten ' 

'(depending on stacker).' 

% (col)) 

else: 

newdtype += ([(col, dtype)]) 

newData = np.empty(simData.shape, dtype=newdtype) 

# Add references to old data. 

for col in simData.dtype.names: 

newData[col] = simData[col] 

# Were all columns present and populated with something not None? If so, then consider 'all there'. 

if sum(cols_present) == len(self.colsAdded): 

cols_present = True 

else: 

cols_present = False 

return newData, cols_present 

 

def run(self, simData): 

""" 

Example: Generate the new stacker columns, given the simdata columns from the database. 

Returns the new simdata structured array that includes the new stacker columns. 

""" 

# Add new columns 

140 ↛ 141line 140 didn't jump to line 141, because the condition on line 140 was never true if len(simData) == 0: 

return simData 

simData, cols_present = self._addStackerCols(simData) 

# Run the method to calculate/add new data. 

try: 

return self._run(simData, cols_present) 

except TypeError: 

warnings.warn('Please update the stacker %s so that the _run method matches the current API. ' 

'This will give you the option to skip re-running stackers if the columns are ' 

'already present.' 

% (self.__class__.__name__)) 

return self._run(simData) 

 

def _run(self, simData, cols_present=False): 

# By moving the calculation of these columns to a separate method, we add the possibility of using 

# stackers with pandas dataframes. The _addStackerCols method won't work with dataframes, but the 

# _run methods are quite likely to (depending on their details), as they are just populating columns. 

raise NotImplementedError('Not Implemented: ' 

'the child stackers should implement their own _run methods')