Coverage for python/lsst/sims/maf/slicers/baseSlicer.py : 19%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import print_function
2from future import standard_library
3standard_library.install_aliases()
4from builtins import str
5from builtins import zip
6from builtins import range
7from builtins import object
8# Base class for all 'Slicer' objects.
9#
10import inspect
11from io import StringIO
12import json
13import warnings
14import numpy as np
15import numpy.ma as ma
16from lsst.sims.maf.utils import getDateVersion
17from future.utils import with_metaclass
19__all__ = ['SlicerRegistry', 'BaseSlicer']
21class SlicerRegistry(type):
22 """
23 Meta class for slicers, to build a registry of slicer classes.
24 """
25 def __init__(cls, name, bases, dict):
26 super(SlicerRegistry, cls).__init__(name, bases, dict)
27 if not hasattr(cls, 'registry'):
28 cls.registry = {}
29 modname = inspect.getmodule(cls).__name__ + '.'
30 if modname.startswith('lsst.sims.maf.slicers'): 30 ↛ 32line 30 didn't jump to line 32, because the condition on line 30 was never false
31 modname = ''
32 slicername = modname + name
33 if slicername in cls.registry: 33 ↛ 34line 33 didn't jump to line 34, because the condition on line 33 was never true
34 raise Exception('Redefining metric %s! (there are >1 slicers with the same name)' %(slicername))
35 if slicername not in ['BaseSlicer', 'BaseSpatialSlicer']:
36 cls.registry[slicername] = cls
37 def getClass(cls, slicername):
38 return cls.registry[slicername]
39 def help(cls, doc=False):
40 for slicername in sorted(cls.registry):
41 if not doc:
42 print(slicername)
43 if doc:
44 print('---- ', slicername, ' ----')
45 print(inspect.getdoc(cls.registry[slicername]))
49class BaseSlicer(with_metaclass(SlicerRegistry, object)):
50 """
51 Base class for all slicers: sets required methods and implements common functionality.
53 After first construction, the slicer should be ready for setupSlicer to define slicePoints, which will
54 let the slicer 'slice' data and generate plots.
55 After init after a restore: everything necessary for using slicer for plotting or
56 saving/restoring metric data should be present (although slicer does not need to be able to
57 slice data again and generally will not be able to).
59 Parameters
60 ----------
61 verbose: boolean, optional
62 True/False flag to send extra output to screen.
63 Default True.
64 badval: int or float, optional
65 The value the Slicer uses to fill masked metric data values
66 Default -666.
67 """
69 def __init__(self, verbose=True, badval=-666):
70 self.verbose = verbose
71 self.badval = badval
72 # Set cacheSize : each slicer will be able to override if appropriate.
73 # Currently only the healpixSlice actually uses the cache: this is set in 'useCache' flag.
74 # If other slicers have the ability to use the cache, they should add this flag and set the
75 # cacheSize in their __init__ methods.
76 self.cacheSize = 0
77 # Set length of Slicer.
78 self.nslice = None
79 self.shape = self.nslice
80 self.slicePoints = {}
81 self.slicerName = self.__class__.__name__
82 self.columnsNeeded = []
83 # Create a dict that saves how to re-init the slicer.
84 # This may not be the whole set of args/kwargs, but those which carry useful metadata or
85 # are absolutely necesary for init.
86 # Will often be overwritten by individual slicer slicer_init dictionaries.
87 self.slicer_init = {'badval':badval}
88 self.plotFuncs = []
89 # Note if the slicer needs OpSim field ID info
90 self.needsFields = False
91 # Set the y-axis range be on the two-d plot
92 if self.nslice is not None:
93 self.spatialExtent = [0,self.nslice-1]
95 def _runMaps(self, maps):
96 """Add map metadata to slicePoints.
97 """
98 if maps is not None:
99 for m in maps:
100 self.slicePoints = m.run(self.slicePoints)
102 def setupSlicer(self, simData, maps=None):
103 """Set up Slicer for data slicing.
105 Set up internal parameters necessary for slicer to slice data and generates indexes on simData.
106 Also sets _sliceSimData for a particular slicer.
108 Parameters
109 -----------
110 simData : np.recarray
111 The simulated data to be sliced.
112 maps : list of lsst.sims.maf.maps objects, optional.
113 Maps to apply at each slicePoint, to add to the slicePoint metadata. Default None.
114 """
115 # Typically args will be simData, but opsimFieldSlicer also uses fieldData.
116 raise NotImplementedError()
119 def getSlicePoints(self):
120 """Return the slicePoint metadata, for all slice points.
121 """
122 return self.slicePoints
124 def __len__(self):
125 """Return nslice, the number of slicePoints in the slicer.
126 """
127 return self.nslice
129 def __iter__(self):
130 """Iterate over the slices.
131 """
132 self.islice = 0
133 return self
135 def __next__(self):
136 """Returns results of self._sliceSimData when iterating over slicer.
138 Results of self._sliceSimData should be dictionary of
139 {'idxs': the data indexes relevant for this slice of the slicer,
140 'slicePoint': the metadata for the slicePoint, which always includes 'sid' key for ID of slicePoint.}
141 """
142 if self.islice >= self.nslice:
143 raise StopIteration
144 islice = self.islice
145 self.islice += 1
146 return self._sliceSimData(islice)
148 def __getitem__(self, islice):
149 return self._sliceSimData(islice)
151 def __eq__(self, otherSlicer):
152 """
153 Evaluate if two slicers are equivalent.
154 """
155 raise NotImplementedError()
157 def __ne__(self, otherSlicer):
158 """
159 Evaluate if two slicers are not equivalent.
160 """
161 if self == otherSlicer:
162 return False
163 else:
164 return True
166 def _sliceSimData(self, slicePoint):
167 """
168 Slice the simulation data appropriately for the slicer.
170 Given the identifying slicePoint metadata
171 The slice of data returned will be the indices of the numpy rec array (the simData)
172 which are appropriate for the metric to be working on, for that slicePoint.
173 """
174 raise NotImplementedError('This method is set up by "setupSlicer" - run that first.')
176 def writeData(self, outfilename, metricValues, metricName='',
177 simDataName ='', constraint=None, metadata='', plotDict=None, displayDict=None):
178 """
179 Save metric values along with the information required to re-build the slicer.
181 Parameters
182 -----------
183 outfilename : str
184 The output file name.
185 metricValues : np.ma.MaskedArray or np.ndarray
186 The metric values to save to disk.
187 """
188 header = {}
189 header['metricName']=metricName
190 header['constraint'] = constraint
191 header['metadata'] = metadata
192 header['simDataName'] = simDataName
193 date, versionInfo = getDateVersion()
194 header['dateRan'] = date
195 if displayDict is None:
196 displayDict = {'group':'Ungrouped'}
197 header['displayDict'] = displayDict
198 header['plotDict'] = plotDict
199 for key in versionInfo:
200 header[key] = versionInfo[key]
201 if hasattr(metricValues, 'mask'): # If it is a masked array
202 data = metricValues.data
203 mask = metricValues.mask
204 fill = metricValues.fill_value
205 else:
206 data = metricValues
207 mask = None
208 fill = None
209 # npz file acts like dictionary: each keyword/value pair below acts as a dictionary in loaded NPZ file.
210 np.savez(outfilename,
211 header = header, # header saved as dictionary
212 metricValues = data, # metric data values
213 mask = mask, # metric mask values
214 fill = fill, # metric badval/fill val
215 slicer_init = self.slicer_init, # dictionary of instantiation parameters
216 slicerName = self.slicerName, # class name
217 slicePoints = self.slicePoints, # slicePoint metadata saved (is a dictionary)
218 slicerNSlice = self.nslice,
219 slicerShape = self.shape)
221 def outputJSON(self, metricValues, metricName='',
222 simDataName ='', metadata='', plotDict=None):
223 """
224 Send metric data to JSON streaming API, along with a little bit of metadata.
226 This method will only work for metrics where the metricDtype is float or int,
227 as JSON will not interpret more complex data properly. These values can't be plotted anyway though.
229 Parameters
230 -----------
231 metricValues : np.ma.MaskedArray or np.ndarray
232 The metric values.
233 metricName : str, optional
234 The name of the metric. Default ''.
235 simDataName : str, optional
236 The name of the simulated data source. Default ''.
237 metadata : str, optional
238 The metadata about this metric. Default ''.
239 plotDict : dict, optional.
240 The plotDict for this metric bundle. Default None.
242 Returns
243 --------
244 StringIO
245 StringIO object containing a header dictionary with metricName/metadata/simDataName/slicerName,
246 and plot labels from plotDict, and metric values/data for plot.
247 if oneDSlicer, the data is [ [bin_left_edge, value], [bin_left_edge, value]..].
248 if a spatial slicer, the data is [ [lon, lat, value], [lon, lat, value] ..].
249 """
250 # Bail if this is not a good data type for JSON.
251 if not (metricValues.dtype == 'float') or (metricValues.dtype == 'int'):
252 warnings.warn('Cannot generate JSON.')
253 io = StringIO()
254 json.dump(['Cannot generate JSON for this file.'], io)
255 return None
256 # Else put everything together for JSON output.
257 if plotDict is None:
258 plotDict = {}
259 plotDict['units'] = ''
260 # Preserve some of the metadata for the plot.
261 header = {}
262 header['metricName'] = metricName
263 header['metadata'] = metadata
264 header['simDataName'] = simDataName
265 header['slicerName'] = self.slicerName
266 header['slicerLen'] = int(self.nslice)
267 # Set some default plot labels if appropriate.
268 if 'title' in plotDict:
269 header['title'] = plotDict['title']
270 else:
271 header['title'] = '%s %s: %s' %(simDataName, metadata, metricName)
272 if 'xlabel' in plotDict:
273 header['xlabel'] = plotDict['xlabel']
274 else:
275 if hasattr(self, 'sliceColName'):
276 header['xlabel'] = '%s (%s)' %(self.sliceColName, self.sliceColUnits)
277 else:
278 header['xlabel'] = '%s' %(metricName)
279 if 'units' in plotDict:
280 header['xlabel'] += ' (%s)' %(plotDict['units'])
281 if 'ylabel' in plotDict:
282 header['ylabel'] = plotDict['ylabel']
283 else:
284 if hasattr(self, 'sliceColName'):
285 header['ylabel'] = '%s' %(metricName)
286 if 'units' in plotDict:
287 header['ylabel'] += ' (%s)' %(plotDict['units'])
288 else:
289 # If it's not a oneDslicer and no ylabel given, don't need one.
290 pass
291 # Bundle up slicer and metric info.
292 metric = []
293 # If metric values is a masked array.
294 if hasattr(metricValues, 'mask'):
295 if 'ra' in self.slicePoints:
296 # Spatial slicer. Translate ra/dec to lon/lat in degrees and output with metric value.
297 for ra, dec, value, mask in zip(self.slicePoints['ra'], self.slicePoints['dec'],
298 metricValues.data, metricValues.mask):
299 if not mask:
300 lon = ra * 180.0/np.pi
301 lat = dec * 180.0/np.pi
302 metric.append([lon, lat, value])
303 elif 'bins' in self.slicePoints:
304 # OneD slicer. Translate bins into bin/left and output with metric value.
305 for i in range(len(metricValues)):
306 binleft = self.slicePoints['bins'][i]
307 value = metricValues.data[i]
308 mask = metricValues.mask[i]
309 if not mask:
310 metric.append([binleft, value])
311 else:
312 metric.append([binleft, 0])
313 metric.append([self.slicePoints['bins'][i+1], 0])
314 elif self.slicerName == 'UniSlicer':
315 metric.append([metricValues[0]])
316 # Else:
317 else:
318 if 'ra' in self.slicePoints:
319 for ra, dec, value in zip(self.slicePoints['ra'], self.slicePoints['dec'], metricValues):
320 lon = ra * 180.0/np.pi
321 lat = dec * 180.0/np.pi
322 metric.append([lon, lat, value])
323 elif 'bins' in self.slicePoints:
324 for i in range(len(metricValues)):
325 binleft = self.slicePoints['bins'][i]
326 value = metricValues[i]
327 metric.append([binleft, value])
328 metric.append(self.slicePoints['bins'][i+1][0])
329 elif self.slicerName == 'UniSlicer':
330 metric.append([metricValues[0]])
331 # Write out JSON output.
332 io = StringIO()
333 json.dump([header, metric], io)
334 return io
336 def readData(self, infilename):
337 """
338 Read metric data from disk, along with the info to rebuild the slicer (minus new slicing capability).
340 Parameters
341 -----------
342 infilename: str
343 The filename containing the metric data.
345 Returns
346 -------
347 np.ma.MaskedArray, lsst.sims.maf.slicer, dict
348 MetricValues stored in data file, the slicer basis for those metric values, and a dictionary
349 containing header information (runName, metadata, etc.).
350 """
351 import lsst.sims.maf.slicers as slicers
352 # Allowing pickles here is required, because otherwise we cannot restore data saved as objects.
353 restored = np.load(infilename, allow_pickle=True)
354 # Get metadata and other simData info.
355 header = restored['header'][()]
356 slicer_init = restored['slicer_init'][()]
357 slicerName = str(restored['slicerName'])
358 slicePoints = restored['slicePoints'][()]
359 # Backwards compatibility issue - map 'spatialkey1/spatialkey2' to 'lonCol/latCol'.
360 if 'spatialkey1' in slicer_init:
361 slicer_init['lonCol'] = slicer_init['spatialkey1']
362 del (slicer_init['spatialkey1'])
363 if 'spatialkey2' in slicer_init:
364 slicer_init['latCol'] = slicer_init['spatialkey2']
365 del (slicer_init['spatialkey2'])
366 try:
367 slicer = getattr(slicers, slicerName)(**slicer_init)
368 except TypeError:
369 warnings.warn('Cannot use saved slicer init values; falling back to defaults')
370 slicer = getattr(slicers, slicerName)()
371 # Restore slicePoint metadata.
372 slicer.nslice = restored['slicerNSlice']
373 slicer.slicePoints = slicePoints
374 slicer.shape = restored['slicerShape']
375 # Get metric data set
376 if restored['mask'][()] is None:
377 metricValues = ma.MaskedArray(data=restored['metricValues'])
378 else:
379 metricValues = ma.MaskedArray(data=restored['metricValues'],
380 mask=restored['mask'],
381 fill_value=restored['fill'])
382 return metricValues, slicer, header