Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

from __future__ import print_function 

from future import standard_library 

standard_library.install_aliases() 

from builtins import str 

from builtins import zip 

from builtins import range 

from builtins import object 

# Base class for all 'Slicer' objects. 

# 

import inspect 

from io import StringIO 

import json 

import warnings 

import numpy as np 

import numpy.ma as ma 

from lsst.sims.maf.utils import getDateVersion 

from future.utils import with_metaclass 

 

__all__ = ['SlicerRegistry', 'BaseSlicer'] 

 

class SlicerRegistry(type): 

""" 

Meta class for slicers, to build a registry of slicer classes. 

""" 

def __init__(cls, name, bases, dict): 

super(SlicerRegistry, cls).__init__(name, bases, dict) 

if not hasattr(cls, 'registry'): 

cls.registry = {} 

modname = inspect.getmodule(cls).__name__ + '.' 

30 ↛ 32line 30 didn't jump to line 32, because the condition on line 30 was never false if modname.startswith('lsst.sims.maf.slicers'): 

modname = '' 

slicername = modname + name 

33 ↛ 34line 33 didn't jump to line 34, because the condition on line 33 was never true if slicername in cls.registry: 

raise Exception('Redefining metric %s! (there are >1 slicers with the same name)' %(slicername)) 

if slicername not in ['BaseSlicer', 'BaseSpatialSlicer']: 

cls.registry[slicername] = cls 

def getClass(cls, slicername): 

return cls.registry[slicername] 

def help(cls, doc=False): 

for slicername in sorted(cls.registry): 

if not doc: 

print(slicername) 

if doc: 

print('---- ', slicername, ' ----') 

print(inspect.getdoc(cls.registry[slicername])) 

 

 

 

class BaseSlicer(with_metaclass(SlicerRegistry, object)): 

""" 

Base class for all slicers: sets required methods and implements common functionality. 

 

After first construction, the slicer should be ready for setupSlicer to define slicePoints, which will 

let the slicer 'slice' data and generate plots. 

After init after a restore: everything necessary for using slicer for plotting or 

saving/restoring metric data should be present (although slicer does not need to be able to 

slice data again and generally will not be able to). 

 

Parameters 

---------- 

verbose: boolean, optional 

True/False flag to send extra output to screen. 

Default True. 

badval: int or float, optional 

The value the Slicer uses to fill masked metric data values 

Default -666. 

""" 

 

def __init__(self, verbose=True, badval=-666): 

self.verbose = verbose 

self.badval = badval 

# Set cacheSize : each slicer will be able to override if appropriate. 

# Currently only the healpixSlice actually uses the cache: this is set in 'useCache' flag. 

# If other slicers have the ability to use the cache, they should add this flag and set the 

# cacheSize in their __init__ methods. 

self.cacheSize = 0 

# Set length of Slicer. 

self.nslice = None 

self.shape = self.nslice 

self.slicePoints = {} 

self.slicerName = self.__class__.__name__ 

self.columnsNeeded = [] 

# Create a dict that saves how to re-init the slicer. 

# This may not be the whole set of args/kwargs, but those which carry useful metadata or 

# are absolutely necesary for init. 

# Will often be overwritten by individual slicer slicer_init dictionaries. 

self.slicer_init = {'badval':badval} 

self.plotFuncs = [] 

# Note if the slicer needs OpSim field ID info 

self.needsFields = False 

# Set the y-axis range be on the two-d plot 

92 ↛ 93line 92 didn't jump to line 93, because the condition on line 92 was never true if self.nslice is not None: 

self.spatialExtent = [0,self.nslice-1] 

 

def _runMaps(self, maps): 

"""Add map metadata to slicePoints. 

""" 

if maps is not None: 

99 ↛ 100line 99 didn't jump to line 100, because the loop on line 99 never started for m in maps: 

self.slicePoints = m.run(self.slicePoints) 

 

def setupSlicer(self, simData, maps=None): 

"""Set up Slicer for data slicing. 

 

Set up internal parameters necessary for slicer to slice data and generates indexes on simData. 

Also sets _sliceSimData for a particular slicer. 

 

Parameters 

----------- 

simData : np.recarray 

The simulated data to be sliced. 

maps : list of lsst.sims.maf.maps objects, optional. 

Maps to apply at each slicePoint, to add to the slicePoint metadata. Default None. 

""" 

# Typically args will be simData, but opsimFieldSlicer also uses fieldData. 

raise NotImplementedError() 

 

 

def getSlicePoints(self): 

"""Return the slicePoint metadata, for all slice points. 

""" 

return self.slicePoints 

 

def __len__(self): 

"""Return nslice, the number of slicePoints in the slicer. 

""" 

return self.nslice 

 

def __iter__(self): 

"""Iterate over the slices. 

""" 

self.islice = 0 

return self 

 

def __next__(self): 

"""Returns results of self._sliceSimData when iterating over slicer. 

 

Results of self._sliceSimData should be dictionary of 

{'idxs': the data indexes relevant for this slice of the slicer, 

'slicePoint': the metadata for the slicePoint, which always includes 'sid' key for ID of slicePoint.} 

""" 

if self.islice >= self.nslice: 

raise StopIteration 

islice = self.islice 

self.islice += 1 

return self._sliceSimData(islice) 

 

def __getitem__(self, islice): 

return self._sliceSimData(islice) 

 

def __eq__(self, otherSlicer): 

""" 

Evaluate if two slicers are equivalent. 

""" 

raise NotImplementedError() 

 

def __ne__(self, otherSlicer): 

""" 

Evaluate if two slicers are not equivalent. 

""" 

if self == otherSlicer: 

return False 

else: 

return True 

 

def _sliceSimData(self, slicePoint): 

""" 

Slice the simulation data appropriately for the slicer. 

 

Given the identifying slicePoint metadata 

The slice of data returned will be the indices of the numpy rec array (the simData) 

which are appropriate for the metric to be working on, for that slicePoint. 

""" 

raise NotImplementedError('This method is set up by "setupSlicer" - run that first.') 

 

def writeData(self, outfilename, metricValues, metricName='', 

simDataName ='', constraint=None, metadata='', plotDict=None, displayDict=None): 

""" 

Save metric values along with the information required to re-build the slicer. 

 

Parameters 

----------- 

outfilename : str 

The output file name. 

metricValues : np.ma.MaskedArray or np.ndarray 

The metric values to save to disk. 

""" 

header = {} 

header['metricName']=metricName 

header['constraint'] = constraint 

header['metadata'] = metadata 

header['simDataName'] = simDataName 

date, versionInfo = getDateVersion() 

header['dateRan'] = date 

if displayDict is None: 

displayDict = {'group':'Ungrouped'} 

header['displayDict'] = displayDict 

header['plotDict'] = plotDict 

for key in versionInfo: 

header[key] = versionInfo[key] 

if hasattr(metricValues, 'mask'): # If it is a masked array 

data = metricValues.data 

mask = metricValues.mask 

fill = metricValues.fill_value 

else: 

data = metricValues 

mask = None 

fill = None 

# npz file acts like dictionary: each keyword/value pair below acts as a dictionary in loaded NPZ file. 

np.savez(outfilename, 

header = header, # header saved as dictionary 

metricValues = data, # metric data values 

mask = mask, # metric mask values 

fill = fill, # metric badval/fill val 

slicer_init = self.slicer_init, # dictionary of instantiation parameters 

slicerName = self.slicerName, # class name 

slicePoints = self.slicePoints, # slicePoint metadata saved (is a dictionary) 

slicerNSlice = self.nslice, 

slicerShape = self.shape) 

 

def outputJSON(self, metricValues, metricName='', 

simDataName ='', metadata='', plotDict=None): 

""" 

Send metric data to JSON streaming API, along with a little bit of metadata. 

 

This method will only work for metrics where the metricDtype is float or int, 

as JSON will not interpret more complex data properly. These values can't be plotted anyway though. 

 

Parameters 

----------- 

metricValues : np.ma.MaskedArray or np.ndarray 

The metric values. 

metricName : str, optional 

The name of the metric. Default ''. 

simDataName : str, optional 

The name of the simulated data source. Default ''. 

metadata : str, optional 

The metadata about this metric. Default ''. 

plotDict : dict, optional. 

The plotDict for this metric bundle. Default None. 

 

Returns 

-------- 

StringIO 

StringIO object containing a header dictionary with metricName/metadata/simDataName/slicerName, 

and plot labels from plotDict, and metric values/data for plot. 

if oneDSlicer, the data is [ [bin_left_edge, value], [bin_left_edge, value]..]. 

if a spatial slicer, the data is [ [lon, lat, value], [lon, lat, value] ..]. 

""" 

# Bail if this is not a good data type for JSON. 

if not (metricValues.dtype == 'float') or (metricValues.dtype == 'int'): 

warnings.warn('Cannot generate JSON.') 

io = StringIO() 

json.dump(['Cannot generate JSON for this file.'], io) 

return None 

# Else put everything together for JSON output. 

if plotDict is None: 

plotDict = {} 

plotDict['units'] = '' 

# Preserve some of the metadata for the plot. 

header = {} 

header['metricName'] = metricName 

header['metadata'] = metadata 

header['simDataName'] = simDataName 

header['slicerName'] = self.slicerName 

header['slicerLen'] = int(self.nslice) 

# Set some default plot labels if appropriate. 

if 'title' in plotDict: 

header['title'] = plotDict['title'] 

else: 

header['title'] = '%s %s: %s' %(simDataName, metadata, metricName) 

if 'xlabel' in plotDict: 

header['xlabel'] = plotDict['xlabel'] 

else: 

if hasattr(self, 'sliceColName'): 

header['xlabel'] = '%s (%s)' %(self.sliceColName, self.sliceColUnits) 

else: 

header['xlabel'] = '%s' %(metricName) 

if 'units' in plotDict: 

header['xlabel'] += ' (%s)' %(plotDict['units']) 

if 'ylabel' in plotDict: 

header['ylabel'] = plotDict['ylabel'] 

else: 

if hasattr(self, 'sliceColName'): 

header['ylabel'] = '%s' %(metricName) 

if 'units' in plotDict: 

header['ylabel'] += ' (%s)' %(plotDict['units']) 

else: 

# If it's not a oneDslicer and no ylabel given, don't need one. 

pass 

# Bundle up slicer and metric info. 

metric = [] 

# If metric values is a masked array. 

if hasattr(metricValues, 'mask'): 

if 'ra' in self.slicePoints: 

# Spatial slicer. Translate ra/dec to lon/lat in degrees and output with metric value. 

for ra, dec, value, mask in zip(self.slicePoints['ra'], self.slicePoints['dec'], 

metricValues.data, metricValues.mask): 

if not mask: 

lon = ra * 180.0/np.pi 

lat = dec * 180.0/np.pi 

metric.append([lon, lat, value]) 

elif 'bins' in self.slicePoints: 

# OneD slicer. Translate bins into bin/left and output with metric value. 

for i in range(len(metricValues)): 

binleft = self.slicePoints['bins'][i] 

value = metricValues.data[i] 

mask = metricValues.mask[i] 

if not mask: 

metric.append([binleft, value]) 

else: 

metric.append([binleft, 0]) 

metric.append([self.slicePoints['bins'][i+1], 0]) 

elif self.slicerName == 'UniSlicer': 

metric.append([metricValues[0]]) 

# Else: 

else: 

if 'ra' in self.slicePoints: 

for ra, dec, value in zip(self.slicePoints['ra'], self.slicePoints['dec'], metricValues): 

lon = ra * 180.0/np.pi 

lat = dec * 180.0/np.pi 

metric.append([lon, lat, value]) 

elif 'bins' in self.slicePoints: 

for i in range(len(metricValues)): 

binleft = self.slicePoints['bins'][i] 

value = metricValues[i] 

metric.append([binleft, value]) 

metric.append(self.slicePoints['bins'][i+1][0]) 

elif self.slicerName == 'UniSlicer': 

metric.append([metricValues[0]]) 

# Write out JSON output. 

io = StringIO() 

json.dump([header, metric], io) 

return io 

 

def readData(self, infilename): 

""" 

Read metric data from disk, along with the info to rebuild the slicer (minus new slicing capability). 

 

Parameters 

----------- 

infilename: str 

The filename containing the metric data. 

 

Returns 

------- 

np.ma.MaskedArray, lsst.sims.maf.slicer, dict 

MetricValues stored in data file, the slicer basis for those metric values, and a dictionary 

containing header information (runName, metadata, etc.). 

""" 

import lsst.sims.maf.slicers as slicers 

# We have many old metric files saved with py2 and these need a bit of extra care to reload. 

# First determine if this is the case: 

restored = np.load(infilename) 

py2_to_py3 = False 

try: 

restored['slicePoints'] 

except UnicodeError: 

# Old metric data files saved by py2 stored the slicepoints with bytes. 

restored = np.load(infilename, encoding='bytes') 

py2_to_py3 = True 

# Get metadata and other simData info. 

header = restored['header'][()] 

slicer_init = restored['slicer_init'][()] 

slicerName = str(restored['slicerName']) 

slicePoints = restored['slicePoints'][()] 

367 ↛ 368line 367 didn't jump to line 368, because the condition on line 367 was never true if py2_to_py3: 

h = {} 

for k, v in header.items(): 

newkey = str(k, 'utf-8') 

if isinstance(v, bytes): 

value = str(v, 'utf-8') 

else: 

value = v 

h[newkey] = value 

header = h 

si = {} 

for k, v in slicer_init.items(): 

newkey = str(k, 'utf-8') 

if isinstance(v, bytes): 

value = str(v, 'utf-8') 

else: 

value = v 

si[newkey] = value 

slicer_init = si 

sp = {} 

for k, v in slicePoints.items(): 

newkey = str(k, 'utf-8') 

sp[newkey] = v 

slicePoints = sp 

slicerName = str(restored['slicerName'], 'utf-8') 

# Backwards compatibility issue - map 'spatialkey1/spatialkey2' to 'lonCol/latCol'. 

393 ↛ 394line 393 didn't jump to line 394, because the condition on line 393 was never true if 'spatialkey1' in slicer_init: 

slicer_init['lonCol'] = slicer_init['spatialkey1'] 

del (slicer_init['spatialkey1']) 

396 ↛ 397line 396 didn't jump to line 397, because the condition on line 396 was never true if 'spatialkey2' in slicer_init: 

slicer_init['latCol'] = slicer_init['spatialkey2'] 

del (slicer_init['spatialkey2']) 

try: 

slicer = getattr(slicers, slicerName)(**slicer_init) 

except TypeError: 

warnings.warn('Cannot use saved slicer init values; falling back to defaults') 

slicer = getattr(slicers, slicerName)() 

# Restore slicePoint metadata. 

slicer.nslice = restored['slicerNSlice'] 

slicer.slicePoints = slicePoints 

slicer.shape = restored['slicerShape'] 

# Get metric data set 

if restored['mask'][()] is None: 

metricValues = ma.MaskedArray(data=restored['metricValues']) 

else: 

metricValues = ma.MaskedArray(data=restored['metricValues'], 

mask=restored['mask'], 

fill_value=restored['fill']) 

return metricValues, slicer, header