Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1import numpy as np 

2from .baseMetric import BaseMetric 

3 

4# A collection of commonly used simple metrics, operating on a single column and returning a float. 

5 

6__all__ = ['PassMetric', 'Coaddm5Metric', 'MaxMetric', 'AbsMaxMetric', 'MeanMetric', 'AbsMeanMetric', 

7 'MedianMetric', 'AbsMedianMetric', 'MinMetric', 'FullRangeMetric', 'RmsMetric', 'SumMetric', 

8 'CountUniqueMetric', 'CountMetric', 'CountRatioMetric', 'CountSubsetMetric', 'RobustRmsMetric', 

9 'MaxPercentMetric', 'AbsMaxPercentMetric', 'BinaryMetric', 'FracAboveMetric', 'FracBelowMetric', 

10 'PercentileMetric', 'NoutliersNsigmaMetric', 'UniqueRatioMetric', 

11 'MeanAngleMetric', 'RmsAngleMetric', 'FullRangeAngleMetric', 'CountExplimMetric'] 

12 

13twopi = 2.0*np.pi 

14 

15 

16class PassMetric(BaseMetric): 

17 """ 

18 Just pass the entire array through 

19 """ 

20 def __init__(self, cols=None, **kwargs): 

21 if cols is None: 

22 cols= [] 

23 super(PassMetric, self).__init__(col=cols, metricDtype='object', **kwargs) 

24 def run(self, dataSlice, slicePoint=None): 

25 return dataSlice 

26 

27 

28class Coaddm5Metric(BaseMetric): 

29 """Calculate the coadded m5 value at this gridpoint. 

30 """ 

31 def __init__(self, m5Col='fiveSigmaDepth', metricName='CoaddM5', **kwargs): 

32 """Instantiate metric. 

33 

34 m5col = the column name of the individual visit m5 data.""" 

35 super(Coaddm5Metric, self).__init__(col=m5Col, metricName=metricName, **kwargs) 

36 

37 def run(self, dataSlice, slicePoint=None): 

38 return 1.25 * np.log10(np.sum(10.**(.8*dataSlice[self.colname]))) 

39 

40class MaxMetric(BaseMetric): 

41 """Calculate the maximum of a simData column slice. 

42 """ 

43 def run(self, dataSlice, slicePoint=None): 

44 return np.max(dataSlice[self.colname]) 

45 

46class AbsMaxMetric(BaseMetric): 

47 """Calculate the max of the absolute value of a simData column slice. 

48 """ 

49 def run(self, dataSlice, slicePoint=None): 

50 return np.max(np.abs(dataSlice[self.colname])) 

51 

52class MeanMetric(BaseMetric): 

53 """Calculate the mean of a simData column slice. 

54 """ 

55 def run(self, dataSlice, slicePoint=None): 

56 return np.mean(dataSlice[self.colname]) 

57 

58class AbsMeanMetric(BaseMetric): 

59 """Calculate the mean of the absolute value of a simData column slice. 

60 """ 

61 def run(self, dataSlice, slicePoint=None): 

62 return np.mean(np.abs(dataSlice[self.colname])) 

63 

64class MedianMetric(BaseMetric): 

65 """Calculate the median of a simData column slice. 

66 """ 

67 def run(self, dataSlice, slicePoint=None): 

68 return np.median(dataSlice[self.colname]) 

69 

70class AbsMedianMetric(BaseMetric): 

71 """Calculate the median of the absolute value of a simData column slice. 

72 """ 

73 def run(self, dataSlice, slicePoint=None): 

74 return np.median(np.abs(dataSlice[self.colname])) 

75 

76class MinMetric(BaseMetric): 

77 """Calculate the minimum of a simData column slice. 

78 """ 

79 def run(self, dataSlice, slicePoint=None): 

80 return np.min(dataSlice[self.colname]) 

81 

82class FullRangeMetric(BaseMetric): 

83 """Calculate the range of a simData column slice. 

84 """ 

85 def run(self, dataSlice, slicePoint=None): 

86 return np.max(dataSlice[self.colname])-np.min(dataSlice[self.colname]) 

87 

88class RmsMetric(BaseMetric): 

89 """Calculate the standard deviation of a simData column slice. 

90 """ 

91 def run(self, dataSlice, slicePoint=None): 

92 return np.std(dataSlice[self.colname]) 

93 

94class SumMetric(BaseMetric): 

95 """Calculate the sum of a simData column slice. 

96 """ 

97 def run(self, dataSlice, slicePoint=None): 

98 return np.sum(dataSlice[self.colname]) 

99 

100class CountUniqueMetric(BaseMetric): 

101 """Return the number of unique values. 

102 """ 

103 def run(self, dataSlice, slicePoint=None): 

104 return np.size(np.unique(dataSlice[self.colname])) 

105 

106class UniqueRatioMetric(BaseMetric): 

107 """Return the number of unique values divided by the total number of values. 

108 """ 

109 def run(self, dataSlice, slicePoint=None): 

110 ntot = float(np.size(dataSlice[self.colname])) 

111 result = np.size(np.unique(dataSlice[self.colname])) / ntot 

112 return result 

113 

114class CountMetric(BaseMetric): 

115 """Count the length of a simData column slice. """ 

116 def __init__(self, col=None, **kwargs): 

117 super(CountMetric, self).__init__(col=col, **kwargs) 

118 self.metricDtype = 'int' 

119 

120 def run(self, dataSlice, slicePoint=None): 

121 return len(dataSlice[self.colname]) 

122 

123 

124class CountExplimMetric(BaseMetric): 

125 """Count the number of x second visits. Useful for rejecting very short exposures 

126 and counting 60s exposures as 2 visits.""" 

127 def __init__(self, col=None, minExp=20., expectedExp=30., expCol='visitExposureTime', **kwargs): 

128 self.minExp = minExp 

129 self.expectedExp = expectedExp 

130 self.expCol = expCol 

131 super().__init__(col=[col, expCol], **kwargs) 

132 self.metricDtype = 'int' 

133 

134 def run(self, dataSlice, slicePoint=None): 

135 nv = dataSlice[self.expCol] / self.expectedExp 

136 nv[np.where(dataSlice[self.expCol] < self.minExp)[0]] = 0 

137 nv = np.round(nv) 

138 return int(np.sum(nv)) 

139 

140class CountRatioMetric(BaseMetric): 

141 """Count the length of a simData column slice, then divide by 'normVal'.  

142 """ 

143 def __init__(self, col=None, normVal=1., metricName=None, **kwargs): 

144 self.normVal = float(normVal) 

145 if metricName is None: 

146 metricName = 'CountRatio %s div %.1f'%(col, normVal) 

147 super(CountRatioMetric, self).__init__(col=col, metricName=metricName, **kwargs) 

148 

149 def run(self, dataSlice, slicePoint=None): 

150 return len(dataSlice[self.colname])/self.normVal 

151 

152class CountSubsetMetric(BaseMetric): 

153 """Count the length of a simData column slice which matches 'subset'.  

154 """ 

155 def __init__(self, col=None, subset=None, **kwargs): 

156 super(CountSubsetMetric, self).__init__(col=col, **kwargs) 

157 self.metricDtype = 'int' 

158 self.badval = 0 

159 self.subset = subset 

160 

161 def run(self, dataSlice, slicePoint=None): 

162 count = len(np.where(dataSlice[self.colname] == self.subset)[0]) 

163 return count 

164 

165class RobustRmsMetric(BaseMetric): 

166 """Use the inter-quartile range of the data to estimate the RMS.  

167 Robust since this calculation does not include outliers in the distribution. 

168 """ 

169 def run(self, dataSlice, slicePoint=None): 

170 iqr = np.percentile(dataSlice[self.colname],75)-np.percentile(dataSlice[self.colname],25) 

171 rms = iqr/1.349 #approximation 

172 return rms 

173 

174class MaxPercentMetric(BaseMetric): 

175 """Return the percent of the data which has the maximum value. 

176 """ 

177 def run(self, dataSlice, slicePoint=None): 

178 nMax = np.size(np.where(dataSlice[self.colname] == np.max(dataSlice[self.colname]))[0]) 

179 percent = nMax / float(dataSlice[self.colname].size) * 100. 

180 return percent 

181 

182class AbsMaxPercentMetric(BaseMetric): 

183 """Return the percent of the data which has the absolute value of the max value of the data. 

184 """ 

185 def run(self, dataSlice, slicePoint=None): 

186 maxVal = np.abs(np.max(dataSlice[self.colname])) 

187 nMax = np.size(np.where(np.abs(dataSlice[self.colname]) == maxVal)[0]) 

188 percent = nMax / float(dataSlice[self.colname].size) * 100.0 

189 return percent 

190 

191class BinaryMetric(BaseMetric): 

192 """Return 1 if there is data.  

193 """ 

194 def run(self, dataSlice, slicePoint=None): 

195 if dataSlice.size > 0: 

196 return 1 

197 else: 

198 return self.badval 

199 

200class FracAboveMetric(BaseMetric): 

201 """Find the fraction of data values above a given value. 

202 """ 

203 def __init__(self, col=None, cutoff=0.5, scale=1, metricName=None, **kwargs): 

204 # Col could just get passed in bundle with kwargs, but by explicitly pulling it out 

205 # first, we support use cases where class instantiated without explicit 'col='). 

206 if metricName is None: 

207 metricName = 'FracAbove %.2f in %s' %(cutoff, col) 

208 super(FracAboveMetric, self).__init__(col, metricName=metricName, **kwargs) 

209 self.cutoff = cutoff 

210 self.scale = scale 

211 def run(self, dataSlice, slicePoint=None): 

212 good = np.where(dataSlice[self.colname] >= self.cutoff)[0] 

213 fracAbove = np.size(good)/float(np.size(dataSlice[self.colname])) 

214 fracAbove = fracAbove * self.scale 

215 return fracAbove 

216 

217class FracBelowMetric(BaseMetric): 

218 """Find the fraction of data values below a given value. 

219 """ 

220 def __init__(self, col=None, cutoff=0.5, scale=1, metricName=None, **kwargs): 

221 if metricName is None: 

222 metricName = 'FracBelow %.2f %s' %(cutoff, col) 

223 super(FracBelowMetric, self).__init__(col, metricName=metricName, **kwargs) 

224 self.cutoff = cutoff 

225 self.scale = scale 

226 def run(self, dataSlice, slicePoint=None): 

227 good = np.where(dataSlice[self.colname] <= self.cutoff)[0] 

228 fracBelow = np.size(good)/float(np.size(dataSlice[self.colname])) 

229 fracBelow = fracBelow * self.scale 

230 return fracBelow 

231 

232class PercentileMetric(BaseMetric): 

233 """Find the value of a column at a given percentile. 

234 """ 

235 def __init__(self, col=None, percentile=90, metricName=None, **kwargs): 

236 if metricName is None: 

237 metricName = '%.0fth%sile %s' %(percentile, '%', col) 

238 super(PercentileMetric, self).__init__(col=col, metricName=metricName, **kwargs) 

239 self.percentile = percentile 

240 def run(self, dataSlice, slicePoint=None): 

241 pval = np.percentile(dataSlice[self.colname], self.percentile) 

242 return pval 

243 

244class NoutliersNsigmaMetric(BaseMetric): 

245 """Calculate the # of visits less than nSigma below the mean (nSigma<0) or 

246 more than nSigma above the mean of 'col'. 

247 """ 

248 def __init__(self, col=None, nSigma=3., metricName=None, **kwargs): 

249 self.nSigma = nSigma 

250 self.col = col 

251 if metricName is None: 

252 metricName = 'Noutliers %.1f %s' %(self.nSigma, self.col) 

253 super(NoutliersNsigmaMetric, self).__init__(col=col, metricName=metricName, **kwargs) 

254 self.metricDtype = 'int' 

255 

256 def run(self, dataSlice, slicePoint=None): 

257 med = np.mean(dataSlice[self.colname]) 

258 std = np.std(dataSlice[self.colname]) 

259 boundary = med + self.nSigma*std 

260 # If nsigma is positive, look for outliers above median. 

261 if self.nSigma >=0: 

262 outsiders = np.where(dataSlice[self.colname] > boundary) 

263 # Else look for outliers below median. 

264 else: 

265 outsiders = np.where(dataSlice[self.colname] < boundary) 

266 return len(dataSlice[self.colname][outsiders]) 

267 

268def _rotateAngles(angles): 

269 """Private utility for the '*Angle' Metrics below. 

270 

271 This takes a series of angles between 0-2pi and rotates them so that the 

272 first angle is at 0, ensuring the biggest 'gap' is at the end of the series. 

273 This simplifies calculations like the 'mean' and 'rms' or 'fullrange', removing 

274 the discontinuity at 0/2pi. 

275 """ 

276 angleidx = np.argsort(angles) 

277 diffangles = np.diff(angles[angleidx]) 

278 start_to_end = np.array([twopi-angles[angleidx][-1] + angles[angleidx][0]], float) 

279 if start_to_end < -2.*np.pi: 

280 raise ValueError('Angular metrics expect radians, this seems to be in degrees') 

281 diffangles = np.concatenate([diffangles, start_to_end]) 

282 maxdiff = np.where(diffangles == diffangles.max())[0] 

283 if len(maxdiff) > 1: 

284 maxdiff = maxdiff[-1:] 

285 if maxdiff == (len(angles)-1): 

286 rotation = angles[angleidx][0] 

287 else: 

288 rotation = angles[angleidx][maxdiff+1][0] 

289 return (rotation, (angles - rotation) % twopi) 

290 

291class MeanAngleMetric(BaseMetric): 

292 """Calculate the mean of an angular (degree) simData column slice. 

293 

294 'MeanAngle' differs from 'Mean' in that it accounts for wraparound at 2pi. 

295 """ 

296 def run(self, dataSlice, slicePoint=None): 

297 """Calculate mean angle via unit vectors. 

298 If unit vector 'strength' is less than 0.1, then just set mean to 180 degrees 

299 (as this indicates nearly uniformly distributed angles).  

300 """ 

301 x = np.cos(np.radians(dataSlice[self.colname])) 

302 y = np.sin(np.radians(dataSlice[self.colname])) 

303 meanx = np.mean(x) 

304 meany = np.mean(y) 

305 angle = np.arctan2(meany, meanx) 

306 radius = np.sqrt(meanx**2 + meany**2) 

307 mean = angle % twopi 

308 if radius < 0.1: 

309 mean = np.pi 

310 return np.degrees(mean) 

311 

312class RmsAngleMetric(BaseMetric): 

313 """Calculate the standard deviation of an angular (degrees) simData column slice. 

314 

315 'RmsAngle' differs from 'Rms' in that it accounts for wraparound at 2pi. 

316 """ 

317 def run(self, dataSlice, slicePoint=None): 

318 rotation, angles = _rotateAngles(np.radians(dataSlice[self.colname])) 

319 return np.std(np.degrees(angles)) 

320 

321class FullRangeAngleMetric(BaseMetric): 

322 """Calculate the full range of an angular (degrees) simData column slice. 

323 

324 'FullRangeAngle' differs from 'FullRange' in that it accounts for wraparound at 2pi. 

325 """ 

326 def run(self, dataSlice, slicePoint=None): 

327 rotation, angles = _rotateAngles(np.radians(dataSlice[self.colname])) 

328 return np.degrees(angles.max() - angles.min())