Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from __future__ import print_function 

2from future import standard_library 

3standard_library.install_aliases() 

4from builtins import str 

5from builtins import zip 

6from builtins import range 

7from builtins import object 

8# Base class for all 'Slicer' objects. 

9# 

10import inspect 

11from io import StringIO 

12import json 

13import warnings 

14import numpy as np 

15import numpy.ma as ma 

16from lsst.sims.maf.utils import getDateVersion 

17from future.utils import with_metaclass 

18 

19__all__ = ['SlicerRegistry', 'BaseSlicer'] 

20 

21class SlicerRegistry(type): 

22 """ 

23 Meta class for slicers, to build a registry of slicer classes. 

24 """ 

25 def __init__(cls, name, bases, dict): 

26 super(SlicerRegistry, cls).__init__(name, bases, dict) 

27 if not hasattr(cls, 'registry'): 

28 cls.registry = {} 

29 modname = inspect.getmodule(cls).__name__ + '.' 

30 if modname.startswith('lsst.sims.maf.slicers'): 30 ↛ 32line 30 didn't jump to line 32, because the condition on line 30 was never false

31 modname = '' 

32 slicername = modname + name 

33 if slicername in cls.registry: 33 ↛ 34line 33 didn't jump to line 34, because the condition on line 33 was never true

34 raise Exception('Redefining metric %s! (there are >1 slicers with the same name)' %(slicername)) 

35 if slicername not in ['BaseSlicer', 'BaseSpatialSlicer']: 

36 cls.registry[slicername] = cls 

37 def getClass(cls, slicername): 

38 return cls.registry[slicername] 

39 def help(cls, doc=False): 

40 for slicername in sorted(cls.registry): 

41 if not doc: 

42 print(slicername) 

43 if doc: 

44 print('---- ', slicername, ' ----') 

45 print(inspect.getdoc(cls.registry[slicername])) 

46 

47 

48 

49class BaseSlicer(with_metaclass(SlicerRegistry, object)): 

50 """ 

51 Base class for all slicers: sets required methods and implements common functionality. 

52 

53 After first construction, the slicer should be ready for setupSlicer to define slicePoints, which will 

54 let the slicer 'slice' data and generate plots. 

55 After init after a restore: everything necessary for using slicer for plotting or 

56 saving/restoring metric data should be present (although slicer does not need to be able to 

57 slice data again and generally will not be able to). 

58 

59 Parameters 

60 ---------- 

61 verbose: boolean, optional 

62 True/False flag to send extra output to screen. 

63 Default True. 

64 badval: int or float, optional 

65 The value the Slicer uses to fill masked metric data values 

66 Default -666. 

67 """ 

68 

69 def __init__(self, verbose=True, badval=-666): 

70 self.verbose = verbose 

71 self.badval = badval 

72 # Set cacheSize : each slicer will be able to override if appropriate. 

73 # Currently only the healpixSlice actually uses the cache: this is set in 'useCache' flag. 

74 # If other slicers have the ability to use the cache, they should add this flag and set the 

75 # cacheSize in their __init__ methods. 

76 self.cacheSize = 0 

77 # Set length of Slicer. 

78 self.nslice = None 

79 self.shape = self.nslice 

80 self.slicePoints = {} 

81 self.slicerName = self.__class__.__name__ 

82 self.columnsNeeded = [] 

83 # Create a dict that saves how to re-init the slicer. 

84 # This may not be the whole set of args/kwargs, but those which carry useful metadata or 

85 # are absolutely necesary for init. 

86 # Will often be overwritten by individual slicer slicer_init dictionaries. 

87 self.slicer_init = {'badval':badval} 

88 self.plotFuncs = [] 

89 # Note if the slicer needs OpSim field ID info 

90 self.needsFields = False 

91 # Set the y-axis range be on the two-d plot 

92 if self.nslice is not None: 

93 self.spatialExtent = [0,self.nslice-1] 

94 

95 def _runMaps(self, maps): 

96 """Add map metadata to slicePoints. 

97 """ 

98 if maps is not None: 

99 for m in maps: 

100 self.slicePoints = m.run(self.slicePoints) 

101 

102 def setupSlicer(self, simData, maps=None): 

103 """Set up Slicer for data slicing. 

104 

105 Set up internal parameters necessary for slicer to slice data and generates indexes on simData. 

106 Also sets _sliceSimData for a particular slicer. 

107 

108 Parameters 

109 ----------- 

110 simData : np.recarray 

111 The simulated data to be sliced. 

112 maps : list of lsst.sims.maf.maps objects, optional. 

113 Maps to apply at each slicePoint, to add to the slicePoint metadata. Default None. 

114 """ 

115 # Typically args will be simData, but opsimFieldSlicer also uses fieldData. 

116 raise NotImplementedError() 

117 

118 

119 def getSlicePoints(self): 

120 """Return the slicePoint metadata, for all slice points. 

121 """ 

122 return self.slicePoints 

123 

124 def __len__(self): 

125 """Return nslice, the number of slicePoints in the slicer. 

126 """ 

127 return self.nslice 

128 

129 def __iter__(self): 

130 """Iterate over the slices. 

131 """ 

132 self.islice = 0 

133 return self 

134 

135 def __next__(self): 

136 """Returns results of self._sliceSimData when iterating over slicer. 

137 

138 Results of self._sliceSimData should be dictionary of 

139 {'idxs': the data indexes relevant for this slice of the slicer, 

140 'slicePoint': the metadata for the slicePoint, which always includes 'sid' key for ID of slicePoint.} 

141 """ 

142 if self.islice >= self.nslice: 

143 raise StopIteration 

144 islice = self.islice 

145 self.islice += 1 

146 return self._sliceSimData(islice) 

147 

148 def __getitem__(self, islice): 

149 return self._sliceSimData(islice) 

150 

151 def __eq__(self, otherSlicer): 

152 """ 

153 Evaluate if two slicers are equivalent. 

154 """ 

155 raise NotImplementedError() 

156 

157 def __ne__(self, otherSlicer): 

158 """ 

159 Evaluate if two slicers are not equivalent. 

160 """ 

161 if self == otherSlicer: 

162 return False 

163 else: 

164 return True 

165 

166 def _sliceSimData(self, slicePoint): 

167 """ 

168 Slice the simulation data appropriately for the slicer. 

169 

170 Given the identifying slicePoint metadata 

171 The slice of data returned will be the indices of the numpy rec array (the simData) 

172 which are appropriate for the metric to be working on, for that slicePoint. 

173 """ 

174 raise NotImplementedError('This method is set up by "setupSlicer" - run that first.') 

175 

176 def writeData(self, outfilename, metricValues, metricName='', 

177 simDataName ='', constraint=None, metadata='', plotDict=None, displayDict=None): 

178 """ 

179 Save metric values along with the information required to re-build the slicer. 

180 

181 Parameters 

182 ----------- 

183 outfilename : str 

184 The output file name. 

185 metricValues : np.ma.MaskedArray or np.ndarray 

186 The metric values to save to disk. 

187 """ 

188 header = {} 

189 header['metricName']=metricName 

190 header['constraint'] = constraint 

191 header['metadata'] = metadata 

192 header['simDataName'] = simDataName 

193 date, versionInfo = getDateVersion() 

194 header['dateRan'] = date 

195 if displayDict is None: 

196 displayDict = {'group':'Ungrouped'} 

197 header['displayDict'] = displayDict 

198 header['plotDict'] = plotDict 

199 for key in versionInfo: 

200 header[key] = versionInfo[key] 

201 if hasattr(metricValues, 'mask'): # If it is a masked array 

202 data = metricValues.data 

203 mask = metricValues.mask 

204 fill = metricValues.fill_value 

205 else: 

206 data = metricValues 

207 mask = None 

208 fill = None 

209 # npz file acts like dictionary: each keyword/value pair below acts as a dictionary in loaded NPZ file. 

210 np.savez(outfilename, 

211 header = header, # header saved as dictionary 

212 metricValues = data, # metric data values 

213 mask = mask, # metric mask values 

214 fill = fill, # metric badval/fill val 

215 slicer_init = self.slicer_init, # dictionary of instantiation parameters 

216 slicerName = self.slicerName, # class name 

217 slicePoints = self.slicePoints, # slicePoint metadata saved (is a dictionary) 

218 slicerNSlice = self.nslice, 

219 slicerShape = self.shape) 

220 

221 def outputJSON(self, metricValues, metricName='', 

222 simDataName ='', metadata='', plotDict=None): 

223 """ 

224 Send metric data to JSON streaming API, along with a little bit of metadata. 

225 

226 This method will only work for metrics where the metricDtype is float or int, 

227 as JSON will not interpret more complex data properly. These values can't be plotted anyway though. 

228 

229 Parameters 

230 ----------- 

231 metricValues : np.ma.MaskedArray or np.ndarray 

232 The metric values. 

233 metricName : str, optional 

234 The name of the metric. Default ''. 

235 simDataName : str, optional 

236 The name of the simulated data source. Default ''. 

237 metadata : str, optional 

238 The metadata about this metric. Default ''. 

239 plotDict : dict, optional. 

240 The plotDict for this metric bundle. Default None. 

241 

242 Returns 

243 -------- 

244 StringIO 

245 StringIO object containing a header dictionary with metricName/metadata/simDataName/slicerName, 

246 and plot labels from plotDict, and metric values/data for plot. 

247 if oneDSlicer, the data is [ [bin_left_edge, value], [bin_left_edge, value]..]. 

248 if a spatial slicer, the data is [ [lon, lat, value], [lon, lat, value] ..]. 

249 """ 

250 # Bail if this is not a good data type for JSON. 

251 if not (metricValues.dtype == 'float') or (metricValues.dtype == 'int'): 

252 warnings.warn('Cannot generate JSON.') 

253 io = StringIO() 

254 json.dump(['Cannot generate JSON for this file.'], io) 

255 return None 

256 # Else put everything together for JSON output. 

257 if plotDict is None: 

258 plotDict = {} 

259 plotDict['units'] = '' 

260 # Preserve some of the metadata for the plot. 

261 header = {} 

262 header['metricName'] = metricName 

263 header['metadata'] = metadata 

264 header['simDataName'] = simDataName 

265 header['slicerName'] = self.slicerName 

266 header['slicerLen'] = int(self.nslice) 

267 # Set some default plot labels if appropriate. 

268 if 'title' in plotDict: 

269 header['title'] = plotDict['title'] 

270 else: 

271 header['title'] = '%s %s: %s' %(simDataName, metadata, metricName) 

272 if 'xlabel' in plotDict: 

273 header['xlabel'] = plotDict['xlabel'] 

274 else: 

275 if hasattr(self, 'sliceColName'): 

276 header['xlabel'] = '%s (%s)' %(self.sliceColName, self.sliceColUnits) 

277 else: 

278 header['xlabel'] = '%s' %(metricName) 

279 if 'units' in plotDict: 

280 header['xlabel'] += ' (%s)' %(plotDict['units']) 

281 if 'ylabel' in plotDict: 

282 header['ylabel'] = plotDict['ylabel'] 

283 else: 

284 if hasattr(self, 'sliceColName'): 

285 header['ylabel'] = '%s' %(metricName) 

286 if 'units' in plotDict: 

287 header['ylabel'] += ' (%s)' %(plotDict['units']) 

288 else: 

289 # If it's not a oneDslicer and no ylabel given, don't need one. 

290 pass 

291 # Bundle up slicer and metric info. 

292 metric = [] 

293 # If metric values is a masked array. 

294 if hasattr(metricValues, 'mask'): 

295 if 'ra' in self.slicePoints: 

296 # Spatial slicer. Translate ra/dec to lon/lat in degrees and output with metric value. 

297 for ra, dec, value, mask in zip(self.slicePoints['ra'], self.slicePoints['dec'], 

298 metricValues.data, metricValues.mask): 

299 if not mask: 

300 lon = ra * 180.0/np.pi 

301 lat = dec * 180.0/np.pi 

302 metric.append([lon, lat, value]) 

303 elif 'bins' in self.slicePoints: 

304 # OneD slicer. Translate bins into bin/left and output with metric value. 

305 for i in range(len(metricValues)): 

306 binleft = self.slicePoints['bins'][i] 

307 value = metricValues.data[i] 

308 mask = metricValues.mask[i] 

309 if not mask: 

310 metric.append([binleft, value]) 

311 else: 

312 metric.append([binleft, 0]) 

313 metric.append([self.slicePoints['bins'][i+1], 0]) 

314 elif self.slicerName == 'UniSlicer': 

315 metric.append([metricValues[0]]) 

316 # Else: 

317 else: 

318 if 'ra' in self.slicePoints: 

319 for ra, dec, value in zip(self.slicePoints['ra'], self.slicePoints['dec'], metricValues): 

320 lon = ra * 180.0/np.pi 

321 lat = dec * 180.0/np.pi 

322 metric.append([lon, lat, value]) 

323 elif 'bins' in self.slicePoints: 

324 for i in range(len(metricValues)): 

325 binleft = self.slicePoints['bins'][i] 

326 value = metricValues[i] 

327 metric.append([binleft, value]) 

328 metric.append(self.slicePoints['bins'][i+1][0]) 

329 elif self.slicerName == 'UniSlicer': 

330 metric.append([metricValues[0]]) 

331 # Write out JSON output. 

332 io = StringIO() 

333 json.dump([header, metric], io) 

334 return io 

335 

336 def readData(self, infilename): 

337 """ 

338 Read metric data from disk, along with the info to rebuild the slicer (minus new slicing capability). 

339 

340 Parameters 

341 ----------- 

342 infilename: str 

343 The filename containing the metric data. 

344 

345 Returns 

346 ------- 

347 np.ma.MaskedArray, lsst.sims.maf.slicer, dict 

348 MetricValues stored in data file, the slicer basis for those metric values, and a dictionary 

349 containing header information (runName, metadata, etc.). 

350 """ 

351 import lsst.sims.maf.slicers as slicers 

352 # Allowing pickles here is required, because otherwise we cannot restore data saved as objects. 

353 restored = np.load(infilename, allow_pickle=True) 

354 # Get metadata and other simData info. 

355 header = restored['header'][()] 

356 slicer_init = restored['slicer_init'][()] 

357 slicerName = str(restored['slicerName']) 

358 slicePoints = restored['slicePoints'][()] 

359 # Backwards compatibility issue - map 'spatialkey1/spatialkey2' to 'lonCol/latCol'. 

360 if 'spatialkey1' in slicer_init: 

361 slicer_init['lonCol'] = slicer_init['spatialkey1'] 

362 del (slicer_init['spatialkey1']) 

363 if 'spatialkey2' in slicer_init: 

364 slicer_init['latCol'] = slicer_init['spatialkey2'] 

365 del (slicer_init['spatialkey2']) 

366 try: 

367 slicer = getattr(slicers, slicerName)(**slicer_init) 

368 except TypeError: 

369 warnings.warn('Cannot use saved slicer init values; falling back to defaults') 

370 slicer = getattr(slicers, slicerName)() 

371 # Restore slicePoint metadata. 

372 slicer.nslice = restored['slicerNSlice'] 

373 slicer.slicePoints = slicePoints 

374 slicer.shape = restored['slicerShape'] 

375 # Get metric data set 

376 if restored['mask'][()] is None: 

377 metricValues = ma.MaskedArray(data=restored['metricValues']) 

378 else: 

379 metricValues = ma.MaskedArray(data=restored['metricValues'], 

380 mask=restored['mask'], 

381 fill_value=restored['fill']) 

382 return metricValues, slicer, header