Coverage for python/lsst/sims/maf/metricBundles/metricBundleGroup.py : 6%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import print_function
2from builtins import object
3import os
4import numpy as np
5import numpy.ma as ma
6import matplotlib.pyplot as plt
7from collections import OrderedDict
9import lsst.sims.maf.db as db
10import lsst.sims.maf.utils as utils
11from lsst.sims.maf.plots import PlotHandler
12import lsst.sims.maf.maps as maps
13from lsst.sims.maf.stackers import BaseDitherStacker
14from .metricBundle import MetricBundle, createEmptyMetricBundle
15import warnings
17__all__ = ['makeBundlesDictFromList', 'MetricBundleGroup']
20def makeBundlesDictFromList(bundleList):
21 """Utility to convert a list of MetricBundles into a dictionary, keyed by the fileRoot names.
23 Raises an exception if the fileroot duplicates another metricBundle.
24 (Note this should alert to potential cases of filename duplication).
26 Parameters
27 ----------
28 bundleList : list of MetricBundles
29 """
30 bDict = {}
31 for b in bundleList:
32 if b.fileRoot in bDict:
33 raise NameError('More than one metricBundle is using the same fileroot, %s' % (b.fileRoot))
34 bDict[b.fileRoot] = b
35 return bDict
38class MetricBundleGroup(object):
39 """The MetricBundleGroup exists to calculate the metric values for a group of
40 MetricBundles.
42 The MetricBundleGroup will query data from a single database table (for multiple
43 constraints), use that data to calculate metric values for multiple slicers,
44 and calculate summary statistics and generate plots for all metrics included in
45 the dictionary passed to the MetricBundleGroup.
47 We calculate the metric values here, rather than in the individual MetricBundles,
48 because it is much more efficient to step through a slicer once (and calculate all
49 the relevant metric values at each point) than it is to repeat this process multiple times.
51 The MetricBundleGroup also determines how to efficiently group the MetricBundles
52 to reduce the number of sql queries of the database, grabbing larger chunks of data at once.
54 Parameters
55 ----------
56 bundleDict : dict or list of MetricBundles
57 Individual MetricBundles should be placed into a dictionary, and then passed to
58 the MetricBundleGroup. The dictionary keys can then be used to identify MetricBundles
59 if needed -- and to identify new MetricBundles which could be created if 'reduce'
60 functions are run on a particular MetricBundle.
61 A bundleDict can be conveniently created from a list of MetricBundles using
62 makeBundlesDictFromList (done automatically if a list is passed in)
63 dbObj : Database
64 The database object (typically an :class:`OpsimDatabase`) connected to the data to be used to
65 calculate metrics.
66 Advanced use: It is possible to set this to None, in which case data should be passed
67 directly to the runCurrent method (and runAll should not be used).
68 outDir : str, opt
69 Directory to save the metric results. Default is the current directory.
70 resultsDb : ResultsDb, opt
71 A results database. If not specified, one will be created in the outDir.
72 This database saves information about the metrics calculated, including their summary statistics.
73 verbose : bool, opt
74 Flag to turn on/off verbose feedback.
75 saveEarly : bool, opt
76 If True, metric values will be saved immediately after they are first calculated (to prevent
77 data loss) as well as after summary statistics are calculated.
78 If False, metric values will only be saved after summary statistics are calculated.
79 dbTable : str, opt
80 The name of the table in the dbObj to query for data.
81 """
82 def __init__(self, bundleDict, dbObj, outDir='.', resultsDb=None, verbose=True,
83 saveEarly=True, dbTable=None):
84 """Set up the MetricBundleGroup.
85 """
86 if type(bundleDict) is list:
87 bundleDict = makeBundlesDictFromList(bundleDict)
88 # Print occasional messages to screen.
89 self.verbose = verbose
90 # Save metric results as soon as possible (in case of crash).
91 self.saveEarly = saveEarly
92 # Check for output directory, create it if needed.
93 self.outDir = outDir
94 if not os.path.isdir(self.outDir):
95 os.makedirs(self.outDir)
97 # Do some type checking on the MetricBundle dictionary.
98 if not isinstance(bundleDict, dict):
99 raise ValueError('bundleDict should be a dictionary containing MetricBundle objects.')
100 for b in bundleDict.values():
101 if not isinstance(b, MetricBundle):
102 raise ValueError('bundleDict should contain only MetricBundle objects.')
103 # Identify the series of constraints.
104 self.constraints = list(set([b.constraint for b in bundleDict.values()]))
105 # Set the bundleDict (all bundles, with all constraints)
106 self.bundleDict = bundleDict
108 # Check the dbObj.
109 if not isinstance(dbObj, db.Database):
110 warnings.warn('Warning: dbObj should be an instantiated Database (or child) object.')
111 self.dbObj = dbObj
112 # Set the table we're going to be querying.
113 self.dbTable = dbTable
114 if self.dbTable is None and self.dbObj is not None:
115 self.dbTable = self.dbObj.defaultTable
117 # Check the resultsDb (optional).
118 if resultsDb is not None:
119 if not isinstance(resultsDb, db.ResultsDb):
120 raise ValueError('resultsDb should be an ResultsDb object')
121 self.resultsDb = resultsDb
123 # Dict to keep track of what's been run:
124 self.hasRun = {}
125 for bk in bundleDict:
126 self.hasRun[bk] = False
128 def _checkCompatible(self, metricBundle1, metricBundle2):
129 """Check if two MetricBundles are "compatible".
130 Compatible indicates that the sql constraints, the slicers, and the maps are the same, and
131 that the stackers do not interfere with each other
132 (i.e. are not trying to set the same column in different ways).
133 Returns True if the MetricBundles are compatible, False if not.
135 Parameters
136 ----------
137 metricBundle1 : MetricBundle
138 metricBundle2 : MetricBundle
140 Returns
141 -------
142 bool
143 """
144 if metricBundle1.constraint != metricBundle2.constraint:
145 return False
146 if metricBundle1.slicer != metricBundle2.slicer:
147 return False
148 if metricBundle1.mapsList.sort() != metricBundle2.mapsList.sort():
149 return False
150 for stacker in metricBundle1.stackerList:
151 for stacker2 in metricBundle2.stackerList:
152 # If the stackers have different names, that's OK, and if they are identical, that's ok.
153 if (stacker.__class__.__name__ == stacker2.__class__.__name__) & (stacker != stacker2):
154 return False
155 # But if we got this far, everything matches.
156 return True
158 def _findCompatibleLists(self):
159 """Find sets of compatible metricBundles from the currentBundleDict.
160 """
161 # CompatibleLists stores a list of lists;
162 # each (nested) list contains the bundleDict _keys_ of a compatible set of metricBundles.
163 #
164 compatibleLists = []
165 for k, b in self.currentBundleDict.items():
166 foundCompatible = False
167 for compatibleList in compatibleLists:
168 comparisonMetricBundleKey = compatibleList[0]
169 compatible = self._checkCompatible(self.bundleDict[comparisonMetricBundleKey], b)
170 if compatible:
171 # Must compare all metricBundles in each subset (if they are a potential match),
172 # as the stackers could be different (and one could be incompatible,
173 # not necessarily the first)
174 for comparisonMetricBundleKey in compatibleList[1:]:
175 compatible = self._checkCompatible(self.bundleDict[comparisonMetricBundleKey], b)
176 if not compatible:
177 # If we find one which is not compatible, stop and go on to the
178 # next subset list.
179 break
180 # Otherwise, we reached the end of the subset and they were all compatible.
181 foundCompatible = True
182 compatibleList.append(k)
183 if not foundCompatible:
184 # Didn't find a pre-existing compatible set; make a new one.
185 compatibleLists.append([k, ])
186 self.compatibleLists = compatibleLists
188 def getData(self, constraint):
189 """Query the data from the database.
191 The currently bundleDict should generally be set before calling getData (using setCurrent).
193 Parameters
194 ----------
195 constraint : str
196 The constraint for the currently active set of MetricBundles.
197 """
198 if self.verbose:
199 if constraint == '':
200 print("Querying database with no constraint.")
201 else:
202 print("Querying database with constraint %s" % (constraint))
203 # Note that we do NOT run the stackers at this point (this must be done in each 'compatible' group).
204 if self.dbTable != 'Summary':
205 distinctExpMJD = False
206 groupBy = None
207 else:
208 distinctExpMJD = True
209 groupBy = 'expMJD'
210 self.simData = utils.getSimData(self.dbObj, constraint, self.dbCols,
211 tableName=self.dbTable, distinctExpMJD=distinctExpMJD,
212 groupBy=groupBy)
214 if self.verbose:
215 print("Found %i visits" % (self.simData.size))
217 # Query for the fieldData if we need it for the opsimFieldSlicer.
218 needFields = [b.slicer.needsFields for b in self.currentBundleDict.values()]
219 if True in needFields:
220 self.fieldData = utils.getFieldData(self.dbObj, constraint)
221 else:
222 self.fieldData = None
224 def runAll(self, clearMemory=False, plotNow=False, plotKwargs=None):
225 """Runs all the metricBundles in the metricBundleGroup, over all constraints.
227 Calculates metric values, then runs reduce functions and summary statistics for
228 all MetricBundles.
230 Parameters
231 ----------
232 clearMemory : bool, opt
233 If True, deletes metric values from memory after running each constraint group.
234 plotNow : bool, opt
235 If True, plots the metric values immediately after calculation.
236 plotKwargs : bool, opt
237 kwargs to pass to plotCurrent.
238 """
239 for constraint in self.constraints:
240 # Set the 'currentBundleDict' which is a dictionary of the metricBundles which match this
241 # constraint.
242 self.setCurrent(constraint)
243 self.runCurrent(constraint, clearMemory=clearMemory,
244 plotNow=plotNow, plotKwargs=plotKwargs)
246 def setCurrent(self, constraint):
247 """Utility to set the currentBundleDict (i.e. a set of metricBundles with the same SQL constraint).
249 Parameters
250 ----------
251 constraint : str
252 The subset of MetricBundles with metricBundle.constraint == constraint will be
253 included in a subset identified as the currentBundleDict.
254 These are the active metrics to be calculated and plotted, etc.
255 """
256 if constraint is None:
257 constraint = ''
258 self.currentBundleDict = {}
259 for k, b in self.bundleDict.items():
260 if b.constraint == constraint:
261 self.currentBundleDict[k] = b
263 def runCurrent(self, constraint, simData=None, clearMemory=False, plotNow=False, plotKwargs=None):
264 """Run all the metricBundles which match this constraint in the metricBundleGroup.
266 Calculates the metric values, then runs reduce functions and summary statistics for
267 metrics in the current set only (see self.setCurrent).
269 Parameters
270 ----------
271 constraint : str
272 constraint to use to set the currently active metrics
273 simData : numpy.ndarray, opt
274 If simData is not None, then this numpy structured array is used instead of querying
275 data from the dbObj.
276 clearMemory : bool, opt
277 If True, metric values are deleted from memory after they are calculated (and saved to disk).
278 plotNow : bool, opt
279 Plot immediately after calculating metric values (instead of the usual procedure, which
280 is to plot after metric values are calculated for all constraints).
281 plotKwargs : kwargs, opt
282 Plotting kwargs to pass to plotCurrent.
283 """
284 # Build list of all the columns needed from the database.
285 self.dbCols = []
286 for b in self.currentBundleDict.values():
287 self.dbCols.extend(b.dbCols)
288 self.dbCols = list(set(self.dbCols))
290 # Can pass simData directly (if had other method for getting data)
291 if simData is not None:
292 self.simData = simData
294 else:
295 self.simData = None
296 # Query for the data.
297 try:
298 self.getData(constraint)
299 except UserWarning:
300 warnings.warn('No data matching constraint %s' % constraint)
301 metricsSkipped = []
302 for b in self.currentBundleDict.values():
303 metricsSkipped.append("%s : %s : %s" % (b.metric.name, b.metadata, b.slicer.slicerName))
304 warnings.warn(' This means skipping metrics %s' % metricsSkipped)
305 return
306 except ValueError:
307 warnings.warn('One or more of the columns requested from the database was not available.' +
308 ' Skipping constraint %s' % constraint)
309 metricsSkipped = []
310 for b in self.currentBundleDict.values():
311 metricsSkipped.append("%s : %s : %s" % (b.metric.name, b.metadata, b.slicer.slicerName))
312 warnings.warn(' This means skipping metrics %s' % metricsSkipped)
313 return
315 # Find compatible subsets of the MetricBundle dictionary,
316 # which can be run/metrics calculated/ together.
317 self._findCompatibleLists()
319 for compatibleList in self.compatibleLists:
320 if self.verbose:
321 print('Running: ', compatibleList)
322 self._runCompatible(compatibleList)
323 if self.verbose:
324 print('Completed metric generation.')
325 for key in compatibleList:
326 self.hasRun[key] = True
327 # Run the reduce methods.
328 if self.verbose:
329 print('Running reduce methods.')
330 self.reduceCurrent()
331 # Run the summary statistics.
332 if self.verbose:
333 print('Running summary statistics.')
334 self.summaryCurrent()
335 if self.verbose:
336 print('Completed.')
337 if plotNow:
338 if plotKwargs is None:
339 self.plotCurrent()
340 else:
341 self.plotCurrent(**plotKwargs)
342 # Optionally: clear results from memory.
343 if clearMemory:
344 for b in self.currentBundleDict.values():
345 b.metricValues = None
346 if self.verbose:
347 print('Deleted metricValues from memory.')
350 def getData(self, constraint):
351 """Query the data from the database.
353 The currently bundleDict should generally be set before calling getData (using setCurrent).
355 Parameters
356 ----------
357 constraint : str
358 The constraint for the currently active set of MetricBundles.
359 """
360 if self.verbose:
361 if constraint == '':
362 print("Querying database %s with no constraint for columns %s." %
363 (self.dbTable, self.dbCols))
364 else:
365 print("Querying database %s with constraint %s for columns %s" %
366 (self.dbTable, constraint, self.dbCols))
367 # Note that we do NOT run the stackers at this point (this must be done in each 'compatible' group).
368 self.simData = utils.getSimData(self.dbObj, constraint, self.dbCols,
369 groupBy='default', tableName=self.dbTable)
371 if self.verbose:
372 print("Found %i visits" % (self.simData.size))
374 # Query for the fieldData if we need it for the opsimFieldSlicer.
375 needFields = [b.slicer.needsFields for b in self.currentBundleDict.values()]
376 if True in needFields:
377 self.fieldData = utils.getFieldData(self.dbObj, constraint)
378 else:
379 self.fieldData = None
382 def _runCompatible(self, compatibleList):
383 """Runs a set of 'compatible' metricbundles in the MetricBundleGroup dictionary,
384 identified by 'compatibleList' keys.
386 A compatible list of MetricBundles is a subset of the currentBundleDict.
387 The currentBundleDict == set of MetricBundles with the same constraint.
388 The compatibleBundles == set of MetricBundles with the same constraint, the same
389 slicer, the same maps applied to the slicer, and stackers which do not clobber each other's data.
391 This is where the work of calculating the metric values is done.
392 """
394 if len(self.simData) == 0:
395 return
397 # Grab a dictionary representation of this subset of the dictionary, for easier iteration.
398 bDict = {key: self.currentBundleDict.get(key) for key in compatibleList}
400 # Find the unique stackers and maps. These are already "compatible" (as id'd by compatibleList).
401 uniqStackers = []
402 allStackers = []
403 uniqMaps = []
404 allMaps = []
405 for b in bDict.values():
406 allStackers += b.stackerList
407 allMaps += b.mapsList
408 for s in allStackers:
409 if s not in uniqStackers:
410 uniqStackers.append(s)
411 for m in allMaps:
412 if m not in uniqMaps:
413 uniqMaps.append(m)
415 # Run stackers.
416 # Run dither stackers first. (this is a bit of a hack -- we should probably figure out
417 # proper hierarchy and DAG so that stackers run in the order they need to. This will catch 90%).
418 ditherStackers = []
419 for s in uniqStackers:
420 if isinstance(s, BaseDitherStacker):
421 ditherStackers.append(s)
422 for stacker in ditherStackers:
423 self.simData = stacker.run(self.simData, override=True)
424 uniqStackers.remove(stacker)
426 for stacker in uniqStackers:
427 # Note that stackers will clobber previously existing rows with the same name.
428 self.simData = stacker.run(self.simData, override=True)
430 # Pull out one of the slicers to use as our 'slicer'.
431 # This will be forced back into all of the metricBundles at the end (so that they track
432 # the same metadata such as the slicePoints, in case the same actual object wasn't used).
433 slicer = list(bDict.values())[0].slicer
434 if (slicer.slicerName == 'OpsimFieldSlicer'):
435 slicer.setupSlicer(self.simData, self.fieldData, maps=uniqMaps)
436 else:
437 slicer.setupSlicer(self.simData, maps=uniqMaps)
438 # Copy the slicer (after setup) back into the individual metricBundles.
439 if slicer.slicerName != 'HealpixSlicer' or slicer.slicerName != 'UniSlicer':
440 for b in bDict.values():
441 b.slicer = slicer
443 # Set up (masked) arrays to store metric data in each metricBundle.
444 for b in bDict.values():
445 b._setupMetricValues()
447 # Set up an ordered dictionary to be the cache if needed:
448 # (Currently using OrderedDict, it might be faster to use 2 regular Dicts instead)
449 if slicer.cacheSize > 0:
450 cacheDict = OrderedDict()
451 cache = True
452 else:
453 cache = False
454 # Run through all slicepoints and calculate metrics.
455 for i, slice_i in enumerate(slicer):
456 slicedata = self.simData[slice_i['idxs']]
457 if len(slicedata) == 0:
458 # No data at this slicepoint. Mask data values.
459 for b in bDict.values():
460 b.metricValues.mask[i] = True
461 else:
462 # There is data! Should we use our data cache?
463 if cache:
464 # Make the data idxs hashable.
465 cacheKey = frozenset(slice_i['idxs'])
466 # If key exists, set flag to use it, otherwise add it
467 if cacheKey in cacheDict:
468 useCache = True
469 cacheVal = cacheDict[cacheKey]
470 # Move this value to the end of the OrderedDict
471 del cacheDict[cacheKey]
472 cacheDict[cacheKey] = cacheVal
473 else:
474 cacheDict[cacheKey] = i
475 useCache = False
476 for b in bDict.values():
477 if useCache:
478 b.metricValues.data[i] = b.metricValues.data[cacheDict[cacheKey]]
479 else:
480 b.metricValues.data[i] = b.metric.run(slicedata, slicePoint=slice_i['slicePoint'])
481 # If we are above the cache size, drop the oldest element from the cache dict.
482 if len(cacheDict) > slicer.cacheSize:
483 del cacheDict[list(cacheDict.keys())[0]]
485 # Not using memoize, just calculate things normally
486 else:
487 for b in bDict.values():
488 b.metricValues.data[i] = b.metric.run(slicedata, slicePoint=slice_i['slicePoint'])
489 # Mask data where metrics could not be computed (according to metric bad value).
490 for b in bDict.values():
491 if b.metricValues.dtype.name == 'object':
492 for ind, val in enumerate(b.metricValues.data):
493 if val is b.metric.badval:
494 b.metricValues.mask[ind] = True
495 else:
496 # For some reason, this doesn't work for dtype=object arrays.
497 b.metricValues.mask = np.where(b.metricValues.data == b.metric.badval,
498 True, b.metricValues.mask)
500 # Save data to disk as we go, although this won't keep summary values, etc. (just failsafe).
501 if self.saveEarly:
502 for b in bDict.values():
503 b.write(outDir=self.outDir, resultsDb=self.resultsDb)
504 else:
505 for b in bDict.values():
506 b.writeDb(resultsDb=self.resultsDb)
508 def reduceAll(self, updateSummaries=True):
509 """Run the reduce methods for all metrics in bundleDict.
511 Running this method, for all MetricBundles at once, assumes that clearMemory was False.
513 Parameters
514 ----------
515 updateSummaries : bool, opt
516 If True, summary metrics are removed from the top-level (non-reduced)
517 MetricBundle. Usually this should be True, as summary metrics are generally
518 intended to run on the simpler data produced by reduce metrics.
519 """
520 for constraint in self.constraints:
521 self.setCurrent(constraint)
522 self.reduceCurrent(updateSummaries=updateSummaries)
524 def reduceCurrent(self, updateSummaries=True):
525 """Run all reduce functions for the metricbundle in the currently active set of MetricBundles.
527 Parameters
528 ----------
529 updateSummaries : bool, opt
530 If True, summary metrics are removed from the top-level (non-reduced)
531 MetricBundle. Usually this should be True, as summary metrics are generally
532 intended to run on the simpler data produced by reduce metrics.
533 """
534 # Create a temporary dictionary to hold the reduced metricbundles.
535 reduceBundleDict = {}
536 for b in self.currentBundleDict.values():
537 # If there are no reduce functions associated with the metric, skip this metricBundle.
538 if len(b.metric.reduceFuncs) > 0:
539 # Apply reduce functions, creating a new metricBundle in the process (new metric values).
540 for reduceFunc in b.metric.reduceFuncs.values():
541 newmetricbundle = b.reduceMetric(reduceFunc)
542 # Add the new metricBundle to our metricBundleGroup dictionary.
543 name = newmetricbundle.metric.name
544 if name in self.bundleDict:
545 name = newmetricbundle.fileRoot
546 reduceBundleDict[name] = newmetricbundle
547 if self.saveEarly:
548 newmetricbundle.write(outDir=self.outDir, resultsDb=self.resultsDb)
549 else:
550 newmetricbundle.writeDb(resultsDb=self.resultsDb)
551 # Remove summaryMetrics from top level metricbundle if desired.
552 if updateSummaries:
553 b.summaryMetrics = []
554 # Add the new metricBundles to the MetricBundleGroup dictionary.
555 self.bundleDict.update(reduceBundleDict)
556 # And add to to the currentBundleDict too, so we run as part of 'summaryCurrent'.
557 self.currentBundleDict.update(reduceBundleDict)
559 def summaryAll(self):
560 """Run the summary statistics for all metrics in bundleDict.
562 Calculating all summary statistics, for all MetricBundles, at this
563 point assumes that clearMemory was False.
564 """
565 for constraint in self.constraints:
566 self.setCurrent(constraint)
567 self.summaryCurrent()
569 def summaryCurrent(self):
570 """Run summary statistics on all the metricBundles in the currently active set of MetricBundles.
571 """
572 for b in self.currentBundleDict.values():
573 b.computeSummaryStats(self.resultsDb)
575 def plotAll(self, savefig=True, outfileSuffix=None, figformat='pdf', dpi=600, trimWhitespace=True,
576 thumbnail=True, closefigs=True):
577 """Generate all the plots for all the metricBundles in bundleDict.
579 Generating all ploots, for all MetricBundles, at this point, assumes that
580 clearMemory was False.
582 Parameters
583 ----------
584 savefig : bool, opt
585 If True, save figures to disk, to self.outDir directory.
586 outfileSuffix : bool, opt
587 Append outfileSuffix to the end of every plot file generated. Useful for generating
588 sequential series of images for movies.
589 figformat : str, opt
590 Matplotlib figure format to use to save to disk. Default pdf.
591 dpi : int, opt
592 DPI for matplotlib figure. Default 600.
593 trimWhitespace : bool, opt
594 If True, trim additional whitespace from final figures. Default True.
595 thumbnail : bool, opt
596 If True, save a small thumbnail jpg version of the output file to disk as well.
597 This is useful for showMaf web pages. Default True.
598 closefigs : bool, opt
599 Close the matplotlib figures after they are saved to disk. If many figures are
600 generated, closing the figures saves significant memory. Default True.
601 """
602 for constraint in self.constraints:
603 if self.verbose:
604 print('Plotting figures with "%s" constraint now.' % (constraint))
606 self.setCurrent(constraint)
607 self.plotCurrent(savefig=savefig, outfileSuffix=outfileSuffix, figformat=figformat, dpi=dpi,
608 trimWhitespace=trimWhitespace, thumbnail=thumbnail, closefigs=closefigs)
610 def plotCurrent(self, savefig=True, outfileSuffix=None, figformat='pdf', dpi=600, trimWhitespace=True,
611 thumbnail=True, closefigs=True):
612 """Generate the plots for the currently active set of MetricBundles.
614 Parameters
615 ----------
616 savefig : bool, opt
617 If True, save figures to disk, to self.outDir directory.
618 outfileSuffix : str, opt
619 Append outfileSuffix to the end of every plot file generated. Useful for generating
620 sequential series of images for movies.
621 figformat : str, opt
622 Matplotlib figure format to use to save to disk. Default pdf.
623 dpi : int, opt
624 DPI for matplotlib figure. Default 600.
625 trimWhitespace : bool, opt
626 If True, trim additional whitespace from final figures. Default True.
627 thumbnail : bool, opt
628 If True, save a small thumbnail jpg version of the output file to disk as well.
629 This is useful for showMaf web pages. Default True.
630 closefigs : bool, opt
631 Close the matplotlib figures after they are saved to disk. If many figures are
632 generated, closing the figures saves significant memory. Default True.
633 """
634 plotHandler = PlotHandler(outDir=self.outDir, resultsDb=self.resultsDb,
635 savefig=savefig, figformat=figformat, dpi=dpi,
636 trimWhitespace=trimWhitespace, thumbnail=thumbnail)
638 for b in self.currentBundleDict.values():
639 try:
640 b.plot(plotHandler=plotHandler, outfileSuffix=outfileSuffix, savefig=savefig)
641 except ValueError as ve:
642 message = 'Plotting failed for metricBundle %s.' % (b.fileRoot)
643 message += ' Error message: %s' % (ve)
644 warnings.warn(message)
645 if closefigs:
646 plt.close('all')
647 if self.verbose:
648 print('Plotting complete.')
650 def writeAll(self):
651 """Save all the MetricBundles to disk.
653 Saving all MetricBundles to disk at this point assumes that clearMemory was False.
654 """
655 for constraint in self.constraints:
656 self.setCurrent(constraint)
657 self.writeCurrent()
659 def writeCurrent(self):
660 """Save all the MetricBundles in the currently active set to disk.
661 """
662 if self.verbose:
663 if self.saveEarly:
664 print('Re-saving metric bundles.')
665 else:
666 print('Saving metric bundles.')
667 for b in self.currentBundleDict.values():
668 b.write(outDir=self.outDir, resultsDb=self.resultsDb)
670 def readAll(self):
671 """Attempt to read all MetricBundles from disk.
673 You must set the metrics/slicer/constraint/runName for a metricBundle appropriately;
674 then this method will search for files in the location self.outDir/metricBundle.fileRoot.
675 Reads all the files associated with all metricbundles in self.bundleDict.
676 """
677 reduceBundleDict = {}
678 removeBundles = []
679 for b in self.bundleDict:
680 bundle = self.bundleDict[b]
681 filename = os.path.join(self.outDir, bundle.fileRoot + '.npz')
682 try:
683 # Create a temporary metricBundle to read the data into.
684 # (we don't use b directly, as this overrides plotDict/etc).
685 tmpBundle = createEmptyMetricBundle()
686 tmpBundle.read(filename)
687 # Copy the tmpBundle metricValues into bundle.
688 bundle.metricValues = tmpBundle.metricValues
689 # And copy the slicer into b, to get slicePoints.
690 bundle.slicer = tmpBundle.slicer
691 if self.verbose:
692 print('Read %s from disk.' % (bundle.fileRoot))
693 except IOError:
694 warnings.warn('Warning: file %s not found, bundle not restored.' % filename)
695 removeBundles.append(b)
697 # Look to see if this is a complex metric, with associated 'reduce' functions,
698 # and read those in too.
699 if len(bundle.metric.reduceFuncs) > 0:
700 origMetricName = bundle.metric.name
701 for reduceFunc in bundle.metric.reduceFuncs.values():
702 reduceName = origMetricName + '_' + reduceFunc.__name__.replace('reduce', '')
703 # Borrow the fileRoot in b (we'll reset it appropriately afterwards).
704 bundle.metric.name = reduceName
705 bundle._buildFileRoot()
706 filename = os.path.join(self.outDir, bundle.fileRoot + '.npz')
707 tmpBundle = createEmptyMetricBundle()
708 try:
709 tmpBundle.read(filename)
710 # This won't necessarily recreate the plotDict and displayDict exactly
711 # as they would have been made if you calculated the reduce metric from scratch.
712 # Perhaps update these metric reduce dictionaries after reading them in?
713 newmetricBundle = MetricBundle(metric=bundle.metric, slicer=bundle.slicer,
714 constraint=bundle.constraint,
715 stackerList=bundle.stackerList, runName=bundle.runName,
716 metadata=bundle.metadata,
717 plotDict=bundle.plotDict,
718 displayDict=bundle.displayDict,
719 summaryMetrics=bundle.summaryMetrics,
720 mapsList=bundle.mapsList,
721 fileRoot=bundle.fileRoot, plotFuncs=bundle.plotFuncs)
722 newmetricBundle.metric.name = reduceName
723 newmetricBundle.metricValues = ma.copy(tmpBundle.metricValues)
724 # Add the new metricBundle to our metricBundleGroup dictionary.
725 name = newmetricBundle.metric.name
726 if name in self.bundleDict:
727 name = newmetricBundle.fileRoot
728 reduceBundleDict[name] = newmetricBundle
729 if self.verbose:
730 print('Read %s from disk.' % (newmetricBundle.fileRoot))
731 except IOError:
732 warnings.warn('Warning: file %s not found, bundle not restored ("reduce" metric).'
733 % filename)
735 # Remove summaryMetrics from top level metricbundle.
736 bundle.summaryMetrics = []
737 # Update parent MetricBundle name.
738 bundle.metric.name = origMetricName
739 bundle._buildFileRoot()
741 # Add the reduce bundles into the bundleDict.
742 self.bundleDict.update(reduceBundleDict)
743 # And remove the bundles which were not found on disk, so we don't try to make (blank) plots.
744 for b in removeBundles:
745 del self.bundleDict[b]