Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from __future__ import print_function 

2from builtins import zip 

3from builtins import object 

4import inspect 

5import warnings 

6import numpy as np 

7from future.utils import with_metaclass 

8 

9__all__ = ['StackerRegistry', 'BaseStacker'] 

10 

11 

12class StackerRegistry(type): 

13 """ 

14 Meta class for Stackers, to build a registry of stacker classes. 

15 """ 

16 

17 def __init__(cls, name, bases, dict): 

18 super(StackerRegistry, cls).__init__(name, bases, dict) 

19 if not hasattr(cls, 'registry'): 

20 cls.registry = {} 

21 if not hasattr(cls, 'sourceDict'): 

22 cls.sourceDict = {} 

23 modname = inspect.getmodule(cls).__name__ 

24 if modname.startswith('lsst.sims.maf.stackers'): 24 ↛ 27line 24 didn't jump to line 27, because the condition on line 24 was never false

25 modname = '' 

26 else: 

27 if len(modname.split('.')) > 1: 

28 modname = '.'.join(modname.split('.')[:-1]) + '.' 

29 else: 

30 modname = modname + '.' 

31 stackername = modname + name 

32 if stackername in cls.registry: 32 ↛ 33line 32 didn't jump to line 33, because the condition on line 32 was never true

33 raise Exception('Redefining stacker %s! (there are >1 stackers with the same name)' 

34 % (stackername)) 

35 if stackername != 'BaseStacker': 

36 cls.registry[stackername] = cls 

37 colsAdded = cls.colsAdded 

38 for col in colsAdded: 

39 cls.sourceDict[col] = cls 

40 

41 def getClass(cls, stackername): 

42 return cls.registry[stackername] 

43 

44 def help(cls, doc=False): 

45 for stackername in sorted(cls.registry): 

46 if not doc: 

47 print(stackername) 

48 if doc: 

49 print('---- ', stackername, ' ----') 

50 print(cls.registry[stackername].__doc__) 

51 stacker = cls.registry[stackername]() 

52 print(' Columns added to SimData: ', ','.join(stacker.colsAdded)) 

53 print(' Default columns required: ', ','.join(stacker.colsReq)) 

54 

55 

56class BaseStacker(with_metaclass(StackerRegistry, object)): 

57 """Base MAF Stacker: add columns generated at run-time to the simdata array.""" 

58 # List of the names of the columns generated by the Stacker. 

59 colsAdded = [] 

60 

61 def __init__(self): 

62 """ 

63 Instantiate the stacker. 

64 This method should be overriden by the user. This serves as an example of 

65 the variables required by the framework. 

66 """ 

67 # Add the list of new columns generated by the stacker as class attributes (colsAdded - above). 

68 # List of the names of the columns required from the database (to generate the Stacker columns). 

69 self.colsReq = [] 

70 # Optional: specify the new column types. 

71 self.colsAddedDtypes = None 

72 # Optional: provide a list of units for the columns defined in colsAdded. 

73 self.units = [None] 

74 

75 def __hash__(self): 

76 return None 

77 

78 def __eq__(self, otherStacker): 

79 """ 

80 Evaluate if two stackers are equivalent. 

81 """ 

82 # If the class names are different, they are not 'the same'. 

83 if self.__class__.__name__ != otherStacker.__class__.__name__: 

84 return False 

85 # Otherwise, this is the same stacker class, but may be instantiated differently. 

86 # We have to delve a little further, and compare the kwargs & attributes for each stacker. 

87 stateNow = dir(self) 

88 for key in stateNow: 

89 if not key.startswith('_') and key != 'registry' and key != 'run' and key != 'next': 

90 if not hasattr(otherStacker, key): 

91 return False 

92 # If the attribute is from numpy, assume it's an array and test it 

93 if type(getattr(self, key)).__module__ == np.__name__: 

94 if not np.array_equal(getattr(self, key), getattr(otherStacker, key)): 

95 return False 

96 else: 

97 if getattr(self, key) != getattr(otherStacker, key): 

98 return False 

99 return True 

100 

101 def __ne__(self, otherStacker): 

102 """ 

103 Evaluate if two stackers are not equal. 

104 """ 

105 if self == otherStacker: 

106 return False 

107 else: 

108 return True 

109 

110 def _addStackerCols(self, simData): 

111 """ 

112 Add the new Stacker columns to the simData array. 

113 If columns already present in simData, just allows 'run' method to overwrite. 

114 Returns simData array with these columns added (so 'run' method can set their values). 

115 """ 

116 if not hasattr(self, 'colsAddedDtypes') or self.colsAddedDtypes is None: 

117 self.colsAddedDtypes = [float for col in self.colsAdded] 

118 # Create description of new recarray. 

119 newdtype = simData.dtype.descr 

120 cols_present = [False] * len(self.colsAdded) 

121 for i, (col, dtype) in enumerate(zip(self.colsAdded, self.colsAddedDtypes)): 

122 if col in simData.dtype.names: 

123 if simData[col][0] is not None: 

124 cols_present[i] = True 

125 warnings.warn('Warning - column %s already present in simData, may be overwritten ' 

126 '(depending on stacker).' 

127 % (col)) 

128 else: 

129 newdtype += ([(col, dtype)]) 

130 newData = np.empty(simData.shape, dtype=newdtype) 

131 # Add references to old data. 

132 for col in simData.dtype.names: 

133 newData[col] = simData[col] 

134 # Were all columns present and populated with something not None? If so, then consider 'all there'. 

135 if sum(cols_present) == len(self.colsAdded): 

136 cols_present = True 

137 else: 

138 cols_present = False 

139 return newData, cols_present 

140 

141 def run(self, simData, override=False): 

142 """ 

143 Example: Generate the new stacker columns, given the simdata columns from the database. 

144 Returns the new simdata structured array that includes the new stacker columns. 

145 """ 

146 # Add new columns 

147 if len(simData) == 0: 

148 return simData 

149 simData, cols_present = self._addStackerCols(simData) 

150 # If override is set, it means go ahead and recalculate stacker values. 

151 if override: 

152 cols_present = False 

153 # Run the method to calculate/add new data. 

154 try: 

155 return self._run(simData, cols_present) 

156 except TypeError: 

157 warnings.warn('Please update the stacker %s so that the _run method matches the current API. ' 

158 'This will give you the option to skip re-running stackers if the columns are ' 

159 'already present.' 

160 % (self.__class__.__name__)) 

161 return self._run(simData) 

162 

163 def _run(self, simData, cols_present=False): 

164 # By moving the calculation of these columns to a separate method, we add the possibility of using 

165 # stackers with pandas dataframes. The _addStackerCols method won't work with dataframes, but the 

166 # _run methods are quite likely to (depending on their details), as they are just populating columns. 

167 raise NotImplementedError('Not Implemented: ' 

168 'the child stackers should implement their own _run methods')