Coverage for python/lsst/sims/maf/metricBundles/metricBundleGroup.py : 6%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import print_function
2from builtins import object
3import os
4import numpy as np
5import numpy.ma as ma
6import matplotlib.pyplot as plt
7from collections import OrderedDict
9import lsst.sims.maf.db as db
10import lsst.sims.maf.utils as utils
11from lsst.sims.maf.plots import PlotHandler
12import lsst.sims.maf.maps as maps
13from lsst.sims.maf.stackers import BaseDitherStacker
14from .metricBundle import MetricBundle, createEmptyMetricBundle
15import warnings
17__all__ = ['makeBundlesDictFromList', 'MetricBundleGroup']
20def makeBundlesDictFromList(bundleList):
21 """Utility to convert a list of MetricBundles into a dictionary, keyed by the fileRoot names.
23 Raises an exception if the fileroot duplicates another metricBundle.
24 (Note this should alert to potential cases of filename duplication).
26 Parameters
27 ----------
28 bundleList : list of MetricBundles
29 """
30 bDict = {}
31 for b in bundleList:
32 if b.fileRoot in bDict:
33 raise NameError('More than one metricBundle is using the same fileroot, %s' % (b.fileRoot))
34 bDict[b.fileRoot] = b
35 return bDict
38class MetricBundleGroup(object):
39 """The MetricBundleGroup exists to calculate the metric values for a group of
40 MetricBundles.
42 The MetricBundleGroup will query data from a single database table (for multiple
43 constraints), use that data to calculate metric values for multiple slicers,
44 and calculate summary statistics and generate plots for all metrics included in
45 the dictionary passed to the MetricBundleGroup.
47 We calculate the metric values here, rather than in the individual MetricBundles,
48 because it is much more efficient to step through a slicer once (and calculate all
49 the relevant metric values at each point) than it is to repeat this process multiple times.
51 The MetricBundleGroup also determines how to efficiently group the MetricBundles
52 to reduce the number of sql queries of the database, grabbing larger chunks of data at once.
54 Parameters
55 ----------
56 bundleDict : dict or list of MetricBundles
57 Individual MetricBundles should be placed into a dictionary, and then passed to
58 the MetricBundleGroup. The dictionary keys can then be used to identify MetricBundles
59 if needed -- and to identify new MetricBundles which could be created if 'reduce'
60 functions are run on a particular MetricBundle.
61 A bundleDict can be conveniently created from a list of MetricBundles using
62 makeBundlesDictFromList (done automatically if a list is passed in)
63 dbObj : Database
64 The database object (typically an :class:`OpsimDatabase`) connected to the data to be used to
65 calculate metrics.
66 Advanced use: It is possible to set this to None, in which case data should be passed
67 directly to the runCurrent method (and runAll should not be used).
68 outDir : str, opt
69 Directory to save the metric results. Default is the current directory.
70 resultsDb : ResultsDb, opt
71 A results database. If not specified, one will be created in the outDir.
72 This database saves information about the metrics calculated, including their summary statistics.
73 verbose : bool, opt
74 Flag to turn on/off verbose feedback.
75 saveEarly : bool, opt
76 If True, metric values will be saved immediately after they are first calculated (to prevent
77 data loss) as well as after summary statistics are calculated.
78 If False, metric values will only be saved after summary statistics are calculated.
79 dbTable : str, opt
80 The name of the table in the dbObj to query for data.
81 """
82 def __init__(self, bundleDict, dbObj, outDir='.', resultsDb=None, verbose=True,
83 saveEarly=True, dbTable=None):
84 """Set up the MetricBundleGroup.
85 """
86 if type(bundleDict) is list:
87 bundleDict = makeBundlesDictFromList(bundleDict)
88 # Print occasional messages to screen.
89 self.verbose = verbose
90 # Save metric results as soon as possible (in case of crash).
91 self.saveEarly = saveEarly
92 # Check for output directory, create it if needed.
93 self.outDir = outDir
94 if not os.path.isdir(self.outDir):
95 os.makedirs(self.outDir)
97 # Do some type checking on the MetricBundle dictionary.
98 if not isinstance(bundleDict, dict):
99 raise ValueError('bundleDict should be a dictionary containing MetricBundle objects.')
100 for b in bundleDict.values():
101 if not isinstance(b, MetricBundle):
102 raise ValueError('bundleDict should contain only MetricBundle objects.')
103 # Identify the series of constraints.
104 self.constraints = list(set([b.constraint for b in bundleDict.values()]))
105 # Set the bundleDict (all bundles, with all constraints)
106 self.bundleDict = bundleDict
108 # Check the dbObj.
109 if not isinstance(dbObj, db.Database):
110 warnings.warn('Warning: dbObj should be an instantiated Database (or child) object.')
111 self.dbObj = dbObj
112 # Set the table we're going to be querying.
113 self.dbTable = dbTable
114 if self.dbTable is None and self.dbObj is not None:
115 self.dbTable = self.dbObj.defaultTable
117 # Check the resultsDb (optional).
118 if resultsDb is not None:
119 if not isinstance(resultsDb, db.ResultsDb):
120 raise ValueError('resultsDb should be an ResultsDb object')
121 self.resultsDb = resultsDb
123 # Dict to keep track of what's been run:
124 self.hasRun = {}
125 for bk in bundleDict:
126 self.hasRun[bk] = False
128 def _checkCompatible(self, metricBundle1, metricBundle2):
129 """Check if two MetricBundles are "compatible".
130 Compatible indicates that the sql constraints, the slicers, and the maps are the same, and
131 that the stackers do not interfere with each other
132 (i.e. are not trying to set the same column in different ways).
133 Returns True if the MetricBundles are compatible, False if not.
135 Parameters
136 ----------
137 metricBundle1 : MetricBundle
138 metricBundle2 : MetricBundle
140 Returns
141 -------
142 bool
143 """
144 if metricBundle1.constraint != metricBundle2.constraint:
145 return False
146 if metricBundle1.slicer != metricBundle2.slicer:
147 return False
148 if metricBundle1.mapsList.sort() != metricBundle2.mapsList.sort():
149 return False
150 for stacker in metricBundle1.stackerList:
151 for stacker2 in metricBundle2.stackerList:
152 # If the stackers have different names, that's OK, and if they are identical, that's ok.
153 if (stacker.__class__.__name__ == stacker2.__class__.__name__) & (stacker != stacker2):
154 return False
155 # But if we got this far, everything matches.
156 return True
158 def _findCompatibleLists(self):
159 """Find sets of compatible metricBundles from the currentBundleDict.
160 """
161 # CompatibleLists stores a list of lists;
162 # each (nested) list contains the bundleDict _keys_ of a compatible set of metricBundles.
163 #
164 compatibleLists = []
165 for k, b in self.currentBundleDict.items():
166 foundCompatible = False
167 for compatibleList in compatibleLists:
168 comparisonMetricBundleKey = compatibleList[0]
169 compatible = self._checkCompatible(self.bundleDict[comparisonMetricBundleKey], b)
170 if compatible:
171 # Must compare all metricBundles in each subset (if they are a potential match),
172 # as the stackers could be different (and one could be incompatible,
173 # not necessarily the first)
174 for comparisonMetricBundleKey in compatibleList[1:]:
175 compatible = self._checkCompatible(self.bundleDict[comparisonMetricBundleKey], b)
176 if not compatible:
177 # If we find one which is not compatible, stop and go on to the
178 # next subset list.
179 break
180 # Otherwise, we reached the end of the subset and they were all compatible.
181 foundCompatible = True
182 compatibleList.append(k)
183 if not foundCompatible:
184 # Didn't find a pre-existing compatible set; make a new one.
185 compatibleLists.append([k, ])
186 self.compatibleLists = compatibleLists
188 def getData(self, constraint):
189 """Query the data from the database.
191 The currently bundleDict should generally be set before calling getData (using setCurrent).
193 Parameters
194 ----------
195 constraint : str
196 The constraint for the currently active set of MetricBundles.
197 """
198 if self.verbose:
199 if constraint == '':
200 print("Querying database with no constraint.")
201 else:
202 print("Querying database with constraint %s" % (constraint))
203 # Note that we do NOT run the stackers at this point (this must be done in each 'compatible' group).
204 if self.dbTable != 'Summary':
205 distinctExpMJD = False
206 groupBy = None
207 else:
208 distinctExpMJD = True
209 groupBy = 'expMJD'
210 self.simData = utils.getSimData(self.dbObj, constraint, self.dbCols,
211 tableName=self.dbTable, distinctExpMJD=distinctExpMJD,
212 groupBy=groupBy)
214 if self.verbose:
215 print("Found %i visits" % (self.simData.size))
217 # Query for the fieldData if we need it for the opsimFieldSlicer.
218 needFields = [b.slicer.needsFields for b in self.currentBundleDict.values()]
219 if True in needFields:
220 self.fieldData = utils.getFieldData(self.dbObj, constraint)
221 else:
222 self.fieldData = None
224 def runAll(self, clearMemory=False, plotNow=False, plotKwargs=None):
225 """Runs all the metricBundles in the metricBundleGroup, over all constraints.
227 Calculates metric values, then runs reduce functions and summary statistics for
228 all MetricBundles.
230 Parameters
231 ----------
232 clearMemory : bool, opt
233 If True, deletes metric values from memory after running each constraint group.
234 plotNow : bool, opt
235 If True, plots the metric values immediately after calculation.
236 plotKwargs : bool, opt
237 kwargs to pass to plotCurrent.
238 """
239 for constraint in self.constraints:
240 # Set the 'currentBundleDict' which is a dictionary of the metricBundles which match this
241 # constraint.
242 self.runCurrent(constraint, clearMemory=clearMemory,
243 plotNow=plotNow, plotKwargs=plotKwargs)
245 def setCurrent(self, constraint):
246 """Utility to set the currentBundleDict (i.e. a set of metricBundles with the same SQL constraint).
248 Parameters
249 ----------
250 constraint : str
251 The subset of MetricBundles with metricBundle.constraint == constraint will be
252 included in a subset identified as the currentBundleDict.
253 These are the active metrics to be calculated and plotted, etc.
254 """
255 if constraint is None:
256 constraint = ''
257 self.currentBundleDict = {}
258 for k, b in self.bundleDict.items():
259 if b.constraint == constraint:
260 self.currentBundleDict[k] = b
262 def runCurrent(self, constraint, simData=None, clearMemory=False, plotNow=False, plotKwargs=None):
263 """Run all the metricBundles which match this constraint in the metricBundleGroup.
265 Calculates the metric values, then runs reduce functions and summary statistics for
266 metrics in the current set only (see self.setCurrent).
268 Parameters
269 ----------
270 constraint : str
271 constraint to use to set the currently active metrics
272 simData : numpy.ndarray, opt
273 If simData is not None, then this numpy structured array is used instead of querying
274 data from the dbObj.
275 clearMemory : bool, opt
276 If True, metric values are deleted from memory after they are calculated (and saved to disk).
277 plotNow : bool, opt
278 Plot immediately after calculating metric values (instead of the usual procedure, which
279 is to plot after metric values are calculated for all constraints).
280 plotKwargs : kwargs, opt
281 Plotting kwargs to pass to plotCurrent.
282 """
283 self.setCurrent(constraint)
285 # Build list of all the columns needed from the database.
286 self.dbCols = []
287 for b in self.currentBundleDict.values():
288 self.dbCols.extend(b.dbCols)
289 self.dbCols = list(set(self.dbCols))
291 # Can pass simData directly (if had other method for getting data)
292 if simData is not None:
293 self.simData = simData
295 else:
296 self.simData = None
297 # Query for the data.
298 try:
299 self.getData(constraint)
300 except UserWarning:
301 warnings.warn('No data matching constraint %s' % constraint)
302 metricsSkipped = []
303 for b in self.currentBundleDict.values():
304 metricsSkipped.append("%s : %s : %s" % (b.metric.name, b.metadata, b.slicer.slicerName))
305 warnings.warn(' This means skipping metrics %s' % metricsSkipped)
306 return
307 except ValueError:
308 warnings.warn('One or more of the columns requested from the database was not available.' +
309 ' Skipping constraint %s' % constraint)
310 metricsSkipped = []
311 for b in self.currentBundleDict.values():
312 metricsSkipped.append("%s : %s : %s" % (b.metric.name, b.metadata, b.slicer.slicerName))
313 warnings.warn(' This means skipping metrics %s' % metricsSkipped)
314 return
316 # Find compatible subsets of the MetricBundle dictionary,
317 # which can be run/metrics calculated/ together.
318 self._findCompatibleLists()
320 for compatibleList in self.compatibleLists:
321 if self.verbose:
322 print('Running: ', compatibleList)
323 self._runCompatible(compatibleList)
324 if self.verbose:
325 print('Completed metric generation.')
326 for key in compatibleList:
327 self.hasRun[key] = True
328 # Run the reduce methods.
329 if self.verbose:
330 print('Running reduce methods.')
331 self.reduceCurrent()
332 # Run the summary statistics.
333 if self.verbose:
334 print('Running summary statistics.')
335 self.summaryCurrent()
336 if self.verbose:
337 print('Completed.')
338 if plotNow:
339 if plotKwargs is None:
340 self.plotCurrent()
341 else:
342 self.plotCurrent(**plotKwargs)
343 # Optionally: clear results from memory.
344 if clearMemory:
345 for b in self.currentBundleDict.values():
346 b.metricValues = None
347 if self.verbose:
348 print('Deleted metricValues from memory.')
351 def getData(self, constraint):
352 """Query the data from the database.
354 The currently bundleDict should generally be set before calling getData (using setCurrent).
356 Parameters
357 ----------
358 constraint : str
359 The constraint for the currently active set of MetricBundles.
360 """
361 if self.verbose:
362 if constraint == '':
363 print("Querying database %s with no constraint for columns %s." %
364 (self.dbTable, self.dbCols))
365 else:
366 print("Querying database %s with constraint %s for columns %s" %
367 (self.dbTable, constraint, self.dbCols))
368 # Note that we do NOT run the stackers at this point (this must be done in each 'compatible' group).
369 self.simData = utils.getSimData(self.dbObj, constraint, self.dbCols,
370 groupBy='default', tableName=self.dbTable)
372 if self.verbose:
373 print("Found %i visits" % (self.simData.size))
375 # Query for the fieldData if we need it for the opsimFieldSlicer.
376 needFields = [b.slicer.needsFields for b in self.currentBundleDict.values()]
377 if True in needFields:
378 self.fieldData = utils.getFieldData(self.dbObj, constraint)
379 else:
380 self.fieldData = None
383 def _runCompatible(self, compatibleList):
384 """Runs a set of 'compatible' metricbundles in the MetricBundleGroup dictionary,
385 identified by 'compatibleList' keys.
387 A compatible list of MetricBundles is a subset of the currentBundleDict.
388 The currentBundleDict == set of MetricBundles with the same constraint.
389 The compatibleBundles == set of MetricBundles with the same constraint, the same
390 slicer, the same maps applied to the slicer, and stackers which do not clobber each other's data.
392 This is where the work of calculating the metric values is done.
393 """
395 if len(self.simData) == 0:
396 return
398 # Grab a dictionary representation of this subset of the dictionary, for easier iteration.
399 bDict = {key: self.currentBundleDict.get(key) for key in compatibleList}
401 # Find the unique stackers and maps. These are already "compatible" (as id'd by compatibleList).
402 uniqStackers = []
403 allStackers = []
404 uniqMaps = []
405 allMaps = []
406 for b in bDict.values():
407 allStackers += b.stackerList
408 allMaps += b.mapsList
409 for s in allStackers:
410 if s not in uniqStackers:
411 uniqStackers.append(s)
412 for m in allMaps:
413 if m not in uniqMaps:
414 uniqMaps.append(m)
416 # Run stackers.
417 # Run dither stackers first. (this is a bit of a hack -- we should probably figure out
418 # proper hierarchy and DAG so that stackers run in the order they need to. This will catch 90%).
419 ditherStackers = []
420 for s in uniqStackers:
421 if isinstance(s, BaseDitherStacker):
422 ditherStackers.append(s)
423 for stacker in ditherStackers:
424 self.simData = stacker.run(self.simData, override=True)
425 uniqStackers.remove(stacker)
427 for stacker in uniqStackers:
428 # Note that stackers will clobber previously existing rows with the same name.
429 self.simData = stacker.run(self.simData, override=True)
431 # Pull out one of the slicers to use as our 'slicer'.
432 # This will be forced back into all of the metricBundles at the end (so that they track
433 # the same metadata such as the slicePoints, in case the same actual object wasn't used).
434 slicer = list(bDict.values())[0].slicer
435 if (slicer.slicerName == 'OpsimFieldSlicer'):
436 slicer.setupSlicer(self.simData, self.fieldData, maps=uniqMaps)
437 else:
438 slicer.setupSlicer(self.simData, maps=uniqMaps)
439 # Copy the slicer (after setup) back into the individual metricBundles.
440 if slicer.slicerName != 'HealpixSlicer' or slicer.slicerName != 'UniSlicer':
441 for b in bDict.values():
442 b.slicer = slicer
444 # Set up (masked) arrays to store metric data in each metricBundle.
445 for b in bDict.values():
446 b._setupMetricValues()
448 # Set up an ordered dictionary to be the cache if needed:
449 # (Currently using OrderedDict, it might be faster to use 2 regular Dicts instead)
450 if slicer.cacheSize > 0:
451 cacheDict = OrderedDict()
452 cache = True
453 else:
454 cache = False
455 # Run through all slicepoints and calculate metrics.
456 for i, slice_i in enumerate(slicer):
457 slicedata = self.simData[slice_i['idxs']]
458 if len(slicedata) == 0:
459 # No data at this slicepoint. Mask data values.
460 for b in bDict.values():
461 b.metricValues.mask[i] = True
462 else:
463 # There is data! Should we use our data cache?
464 if cache:
465 # Make the data idxs hashable.
466 cacheKey = frozenset(slice_i['idxs'])
467 # If key exists, set flag to use it, otherwise add it
468 if cacheKey in cacheDict:
469 useCache = True
470 cacheVal = cacheDict[cacheKey]
471 # Move this value to the end of the OrderedDict
472 del cacheDict[cacheKey]
473 cacheDict[cacheKey] = cacheVal
474 else:
475 cacheDict[cacheKey] = i
476 useCache = False
477 for b in bDict.values():
478 if useCache:
479 b.metricValues.data[i] = b.metricValues.data[cacheDict[cacheKey]]
480 else:
481 b.metricValues.data[i] = b.metric.run(slicedata, slicePoint=slice_i['slicePoint'])
482 # If we are above the cache size, drop the oldest element from the cache dict.
483 if len(cacheDict) > slicer.cacheSize:
484 del cacheDict[list(cacheDict.keys())[0]]
486 # Not using memoize, just calculate things normally
487 else:
488 for b in bDict.values():
489 b.metricValues.data[i] = b.metric.run(slicedata, slicePoint=slice_i['slicePoint'])
490 # Mask data where metrics could not be computed (according to metric bad value).
491 for b in bDict.values():
492 if b.metricValues.dtype.name == 'object':
493 for ind, val in enumerate(b.metricValues.data):
494 if val is b.metric.badval:
495 b.metricValues.mask[ind] = True
496 else:
497 # For some reason, this doesn't work for dtype=object arrays.
498 b.metricValues.mask = np.where(b.metricValues.data == b.metric.badval,
499 True, b.metricValues.mask)
501 # Save data to disk as we go, although this won't keep summary values, etc. (just failsafe).
502 if self.saveEarly:
503 for b in bDict.values():
504 b.write(outDir=self.outDir, resultsDb=self.resultsDb)
505 else:
506 for b in bDict.values():
507 b.writeDb(resultsDb=self.resultsDb)
509 def reduceAll(self, updateSummaries=True):
510 """Run the reduce methods for all metrics in bundleDict.
512 Running this method, for all MetricBundles at once, assumes that clearMemory was False.
514 Parameters
515 ----------
516 updateSummaries : bool, opt
517 If True, summary metrics are removed from the top-level (non-reduced)
518 MetricBundle. Usually this should be True, as summary metrics are generally
519 intended to run on the simpler data produced by reduce metrics.
520 """
521 for constraint in self.constraints:
522 self.setCurrent(constraint)
523 self.reduceCurrent(updateSummaries=updateSummaries)
525 def reduceCurrent(self, updateSummaries=True):
526 """Run all reduce functions for the metricbundle in the currently active set of MetricBundles.
528 Parameters
529 ----------
530 updateSummaries : bool, opt
531 If True, summary metrics are removed from the top-level (non-reduced)
532 MetricBundle. Usually this should be True, as summary metrics are generally
533 intended to run on the simpler data produced by reduce metrics.
534 """
535 # Create a temporary dictionary to hold the reduced metricbundles.
536 reduceBundleDict = {}
537 for b in self.currentBundleDict.values():
538 # If there are no reduce functions associated with the metric, skip this metricBundle.
539 if len(b.metric.reduceFuncs) > 0:
540 # Apply reduce functions, creating a new metricBundle in the process (new metric values).
541 for reduceFunc in b.metric.reduceFuncs.values():
542 newmetricbundle = b.reduceMetric(reduceFunc)
543 # Add the new metricBundle to our metricBundleGroup dictionary.
544 name = newmetricbundle.metric.name
545 if name in self.bundleDict:
546 name = newmetricbundle.fileRoot
547 reduceBundleDict[name] = newmetricbundle
548 if self.saveEarly:
549 newmetricbundle.write(outDir=self.outDir, resultsDb=self.resultsDb)
550 else:
551 newmetricbundle.writeDb(resultsDb=self.resultsDb)
552 # Remove summaryMetrics from top level metricbundle if desired.
553 if updateSummaries:
554 b.summaryMetrics = []
555 # Add the new metricBundles to the MetricBundleGroup dictionary.
556 self.bundleDict.update(reduceBundleDict)
557 # And add to to the currentBundleDict too, so we run as part of 'summaryCurrent'.
558 self.currentBundleDict.update(reduceBundleDict)
560 def summaryAll(self):
561 """Run the summary statistics for all metrics in bundleDict.
563 Calculating all summary statistics, for all MetricBundles, at this
564 point assumes that clearMemory was False.
565 """
566 for constraint in self.constraints:
567 self.setCurrent(constraint)
568 self.summaryCurrent()
570 def summaryCurrent(self):
571 """Run summary statistics on all the metricBundles in the currently active set of MetricBundles.
572 """
573 for b in self.currentBundleDict.values():
574 b.computeSummaryStats(self.resultsDb)
576 def plotAll(self, savefig=True, outfileSuffix=None, figformat='pdf', dpi=600, trimWhitespace=True,
577 thumbnail=True, closefigs=True):
578 """Generate all the plots for all the metricBundles in bundleDict.
580 Generating all ploots, for all MetricBundles, at this point, assumes that
581 clearMemory was False.
583 Parameters
584 ----------
585 savefig : bool, opt
586 If True, save figures to disk, to self.outDir directory.
587 outfileSuffix : bool, opt
588 Append outfileSuffix to the end of every plot file generated. Useful for generating
589 sequential series of images for movies.
590 figformat : str, opt
591 Matplotlib figure format to use to save to disk. Default pdf.
592 dpi : int, opt
593 DPI for matplotlib figure. Default 600.
594 trimWhitespace : bool, opt
595 If True, trim additional whitespace from final figures. Default True.
596 thumbnail : bool, opt
597 If True, save a small thumbnail jpg version of the output file to disk as well.
598 This is useful for showMaf web pages. Default True.
599 closefigs : bool, opt
600 Close the matplotlib figures after they are saved to disk. If many figures are
601 generated, closing the figures saves significant memory. Default True.
602 """
603 for constraint in self.constraints:
604 if self.verbose:
605 print('Plotting figures with "%s" constraint now.' % (constraint))
607 self.setCurrent(constraint)
608 self.plotCurrent(savefig=savefig, outfileSuffix=outfileSuffix, figformat=figformat, dpi=dpi,
609 trimWhitespace=trimWhitespace, thumbnail=thumbnail, closefigs=closefigs)
611 def plotCurrent(self, savefig=True, outfileSuffix=None, figformat='pdf', dpi=600, trimWhitespace=True,
612 thumbnail=True, closefigs=True):
613 """Generate the plots for the currently active set of MetricBundles.
615 Parameters
616 ----------
617 savefig : bool, opt
618 If True, save figures to disk, to self.outDir directory.
619 outfileSuffix : str, opt
620 Append outfileSuffix to the end of every plot file generated. Useful for generating
621 sequential series of images for movies.
622 figformat : str, opt
623 Matplotlib figure format to use to save to disk. Default pdf.
624 dpi : int, opt
625 DPI for matplotlib figure. Default 600.
626 trimWhitespace : bool, opt
627 If True, trim additional whitespace from final figures. Default True.
628 thumbnail : bool, opt
629 If True, save a small thumbnail jpg version of the output file to disk as well.
630 This is useful for showMaf web pages. Default True.
631 closefigs : bool, opt
632 Close the matplotlib figures after they are saved to disk. If many figures are
633 generated, closing the figures saves significant memory. Default True.
634 """
635 plotHandler = PlotHandler(outDir=self.outDir, resultsDb=self.resultsDb,
636 savefig=savefig, figformat=figformat, dpi=dpi,
637 trimWhitespace=trimWhitespace, thumbnail=thumbnail)
639 for b in self.currentBundleDict.values():
640 try:
641 b.plot(plotHandler=plotHandler, outfileSuffix=outfileSuffix, savefig=savefig)
642 except ValueError as ve:
643 message = 'Plotting failed for metricBundle %s.' % (b.fileRoot)
644 message += ' Error message: %s' % (ve)
645 warnings.warn(message)
646 if closefigs:
647 plt.close('all')
648 if self.verbose:
649 print('Plotting complete.')
651 def writeAll(self):
652 """Save all the MetricBundles to disk.
654 Saving all MetricBundles to disk at this point assumes that clearMemory was False.
655 """
656 for constraint in self.constraints:
657 self.setCurrent(constraint)
658 self.writeCurrent()
660 def writeCurrent(self):
661 """Save all the MetricBundles in the currently active set to disk.
662 """
663 if self.verbose:
664 if self.saveEarly:
665 print('Re-saving metric bundles.')
666 else:
667 print('Saving metric bundles.')
668 for b in self.currentBundleDict.values():
669 b.write(outDir=self.outDir, resultsDb=self.resultsDb)
671 def readAll(self):
672 """Attempt to read all MetricBundles from disk.
674 You must set the metrics/slicer/constraint/runName for a metricBundle appropriately;
675 then this method will search for files in the location self.outDir/metricBundle.fileRoot.
676 Reads all the files associated with all metricbundles in self.bundleDict.
677 """
678 reduceBundleDict = {}
679 removeBundles = []
680 for b in self.bundleDict:
681 bundle = self.bundleDict[b]
682 filename = os.path.join(self.outDir, bundle.fileRoot + '.npz')
683 try:
684 # Create a temporary metricBundle to read the data into.
685 # (we don't use b directly, as this overrides plotDict/etc).
686 tmpBundle = createEmptyMetricBundle()
687 tmpBundle.read(filename)
688 # Copy the tmpBundle metricValues into bundle.
689 bundle.metricValues = tmpBundle.metricValues
690 # And copy the slicer into b, to get slicePoints.
691 bundle.slicer = tmpBundle.slicer
692 if self.verbose:
693 print('Read %s from disk.' % (bundle.fileRoot))
694 except IOError:
695 warnings.warn('Warning: file %s not found, bundle not restored.' % filename)
696 removeBundles.append(b)
698 # Look to see if this is a complex metric, with associated 'reduce' functions,
699 # and read those in too.
700 if len(bundle.metric.reduceFuncs) > 0:
701 origMetricName = bundle.metric.name
702 for reduceFunc in bundle.metric.reduceFuncs.values():
703 reduceName = origMetricName + '_' + reduceFunc.__name__.replace('reduce', '')
704 # Borrow the fileRoot in b (we'll reset it appropriately afterwards).
705 bundle.metric.name = reduceName
706 bundle._buildFileRoot()
707 filename = os.path.join(self.outDir, bundle.fileRoot + '.npz')
708 tmpBundle = createEmptyMetricBundle()
709 try:
710 tmpBundle.read(filename)
711 # This won't necessarily recreate the plotDict and displayDict exactly
712 # as they would have been made if you calculated the reduce metric from scratch.
713 # Perhaps update these metric reduce dictionaries after reading them in?
714 newmetricBundle = MetricBundle(metric=bundle.metric, slicer=bundle.slicer,
715 constraint=bundle.constraint,
716 stackerList=bundle.stackerList, runName=bundle.runName,
717 metadata=bundle.metadata,
718 plotDict=bundle.plotDict,
719 displayDict=bundle.displayDict,
720 summaryMetrics=bundle.summaryMetrics,
721 mapsList=bundle.mapsList,
722 fileRoot=bundle.fileRoot, plotFuncs=bundle.plotFuncs)
723 newmetricBundle.metric.name = reduceName
724 newmetricBundle.metricValues = ma.copy(tmpBundle.metricValues)
725 # Add the new metricBundle to our metricBundleGroup dictionary.
726 name = newmetricBundle.metric.name
727 if name in self.bundleDict:
728 name = newmetricBundle.fileRoot
729 reduceBundleDict[name] = newmetricBundle
730 if self.verbose:
731 print('Read %s from disk.' % (newmetricBundle.fileRoot))
732 except IOError:
733 warnings.warn('Warning: file %s not found, bundle not restored ("reduce" metric).'
734 % filename)
736 # Remove summaryMetrics from top level metricbundle.
737 bundle.summaryMetrics = []
738 # Update parent MetricBundle name.
739 bundle.metric.name = origMetricName
740 bundle._buildFileRoot()
742 # Add the reduce bundles into the bundleDict.
743 self.bundleDict.update(reduceBundleDict)
744 # And remove the bundles which were not found on disk, so we don't try to make (blank) plots.
745 for b in removeBundles:
746 del self.bundleDict[b]