Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from __future__ import print_function 

2from builtins import object 

3import os 

4import numpy as np 

5import numpy.ma as ma 

6import matplotlib.pyplot as plt 

7from collections import OrderedDict 

8 

9import lsst.sims.maf.db as db 

10import lsst.sims.maf.utils as utils 

11from lsst.sims.maf.plots import PlotHandler 

12import lsst.sims.maf.maps as maps 

13from lsst.sims.maf.stackers import BaseDitherStacker 

14from .metricBundle import MetricBundle, createEmptyMetricBundle 

15import warnings 

16 

17__all__ = ['makeBundlesDictFromList', 'MetricBundleGroup'] 

18 

19 

20def makeBundlesDictFromList(bundleList): 

21 """Utility to convert a list of MetricBundles into a dictionary, keyed by the fileRoot names. 

22 

23 Raises an exception if the fileroot duplicates another metricBundle. 

24 (Note this should alert to potential cases of filename duplication). 

25 

26 Parameters 

27 ---------- 

28 bundleList : list of MetricBundles 

29 """ 

30 bDict = {} 

31 for b in bundleList: 

32 if b.fileRoot in bDict: 

33 raise NameError('More than one metricBundle is using the same fileroot, %s' % (b.fileRoot)) 

34 bDict[b.fileRoot] = b 

35 return bDict 

36 

37 

38class MetricBundleGroup(object): 

39 """The MetricBundleGroup exists to calculate the metric values for a group of 

40 MetricBundles. 

41 

42 The MetricBundleGroup will query data from a single database table (for multiple 

43 constraints), use that data to calculate metric values for multiple slicers, 

44 and calculate summary statistics and generate plots for all metrics included in 

45 the dictionary passed to the MetricBundleGroup. 

46 

47 We calculate the metric values here, rather than in the individual MetricBundles, 

48 because it is much more efficient to step through a slicer once (and calculate all 

49 the relevant metric values at each point) than it is to repeat this process multiple times. 

50 

51 The MetricBundleGroup also determines how to efficiently group the MetricBundles 

52 to reduce the number of sql queries of the database, grabbing larger chunks of data at once. 

53 

54 Parameters 

55 ---------- 

56 bundleDict : dict or list of MetricBundles 

57 Individual MetricBundles should be placed into a dictionary, and then passed to 

58 the MetricBundleGroup. The dictionary keys can then be used to identify MetricBundles 

59 if needed -- and to identify new MetricBundles which could be created if 'reduce' 

60 functions are run on a particular MetricBundle. 

61 A bundleDict can be conveniently created from a list of MetricBundles using 

62 makeBundlesDictFromList (done automatically if a list is passed in) 

63 dbObj : Database 

64 The database object (typically an :class:`OpsimDatabase`) connected to the data to be used to 

65 calculate metrics. 

66 Advanced use: It is possible to set this to None, in which case data should be passed 

67 directly to the runCurrent method (and runAll should not be used). 

68 outDir : str, opt 

69 Directory to save the metric results. Default is the current directory. 

70 resultsDb : ResultsDb, opt 

71 A results database. If not specified, one will be created in the outDir. 

72 This database saves information about the metrics calculated, including their summary statistics. 

73 verbose : bool, opt 

74 Flag to turn on/off verbose feedback. 

75 saveEarly : bool, opt 

76 If True, metric values will be saved immediately after they are first calculated (to prevent 

77 data loss) as well as after summary statistics are calculated. 

78 If False, metric values will only be saved after summary statistics are calculated. 

79 dbTable : str, opt 

80 The name of the table in the dbObj to query for data. 

81 """ 

82 def __init__(self, bundleDict, dbObj, outDir='.', resultsDb=None, verbose=True, 

83 saveEarly=True, dbTable=None): 

84 """Set up the MetricBundleGroup. 

85 """ 

86 if type(bundleDict) is list: 

87 bundleDict = makeBundlesDictFromList(bundleDict) 

88 # Print occasional messages to screen. 

89 self.verbose = verbose 

90 # Save metric results as soon as possible (in case of crash). 

91 self.saveEarly = saveEarly 

92 # Check for output directory, create it if needed. 

93 self.outDir = outDir 

94 if not os.path.isdir(self.outDir): 

95 os.makedirs(self.outDir) 

96 

97 # Do some type checking on the MetricBundle dictionary. 

98 if not isinstance(bundleDict, dict): 

99 raise ValueError('bundleDict should be a dictionary containing MetricBundle objects.') 

100 for b in bundleDict.values(): 

101 if not isinstance(b, MetricBundle): 

102 raise ValueError('bundleDict should contain only MetricBundle objects.') 

103 # Identify the series of constraints. 

104 self.constraints = list(set([b.constraint for b in bundleDict.values()])) 

105 # Set the bundleDict (all bundles, with all constraints) 

106 self.bundleDict = bundleDict 

107 

108 # Check the dbObj. 

109 if not isinstance(dbObj, db.Database): 

110 warnings.warn('Warning: dbObj should be an instantiated Database (or child) object.') 

111 self.dbObj = dbObj 

112 # Set the table we're going to be querying. 

113 self.dbTable = dbTable 

114 if self.dbTable is None and self.dbObj is not None: 

115 self.dbTable = self.dbObj.defaultTable 

116 

117 # Check the resultsDb (optional). 

118 if resultsDb is not None: 

119 if not isinstance(resultsDb, db.ResultsDb): 

120 raise ValueError('resultsDb should be an ResultsDb object') 

121 self.resultsDb = resultsDb 

122 

123 # Dict to keep track of what's been run: 

124 self.hasRun = {} 

125 for bk in bundleDict: 

126 self.hasRun[bk] = False 

127 

128 def _checkCompatible(self, metricBundle1, metricBundle2): 

129 """Check if two MetricBundles are "compatible". 

130 Compatible indicates that the sql constraints, the slicers, and the maps are the same, and 

131 that the stackers do not interfere with each other 

132 (i.e. are not trying to set the same column in different ways). 

133 Returns True if the MetricBundles are compatible, False if not. 

134 

135 Parameters 

136 ---------- 

137 metricBundle1 : MetricBundle 

138 metricBundle2 : MetricBundle 

139 

140 Returns 

141 ------- 

142 bool 

143 """ 

144 if metricBundle1.constraint != metricBundle2.constraint: 

145 return False 

146 if metricBundle1.slicer != metricBundle2.slicer: 

147 return False 

148 if metricBundle1.mapsList.sort() != metricBundle2.mapsList.sort(): 

149 return False 

150 for stacker in metricBundle1.stackerList: 

151 for stacker2 in metricBundle2.stackerList: 

152 # If the stackers have different names, that's OK, and if they are identical, that's ok. 

153 if (stacker.__class__.__name__ == stacker2.__class__.__name__) & (stacker != stacker2): 

154 return False 

155 # But if we got this far, everything matches. 

156 return True 

157 

158 def _findCompatibleLists(self): 

159 """Find sets of compatible metricBundles from the currentBundleDict. 

160 """ 

161 # CompatibleLists stores a list of lists; 

162 # each (nested) list contains the bundleDict _keys_ of a compatible set of metricBundles. 

163 # 

164 compatibleLists = [] 

165 for k, b in self.currentBundleDict.items(): 

166 foundCompatible = False 

167 for compatibleList in compatibleLists: 

168 comparisonMetricBundleKey = compatibleList[0] 

169 compatible = self._checkCompatible(self.bundleDict[comparisonMetricBundleKey], b) 

170 if compatible: 

171 # Must compare all metricBundles in each subset (if they are a potential match), 

172 # as the stackers could be different (and one could be incompatible, 

173 # not necessarily the first) 

174 for comparisonMetricBundleKey in compatibleList[1:]: 

175 compatible = self._checkCompatible(self.bundleDict[comparisonMetricBundleKey], b) 

176 if not compatible: 

177 # If we find one which is not compatible, stop and go on to the 

178 # next subset list. 

179 break 

180 # Otherwise, we reached the end of the subset and they were all compatible. 

181 foundCompatible = True 

182 compatibleList.append(k) 

183 if not foundCompatible: 

184 # Didn't find a pre-existing compatible set; make a new one. 

185 compatibleLists.append([k, ]) 

186 self.compatibleLists = compatibleLists 

187 

188 def getData(self, constraint): 

189 """Query the data from the database. 

190 

191 The currently bundleDict should generally be set before calling getData (using setCurrent). 

192 

193 Parameters 

194 ---------- 

195 constraint : str 

196 The constraint for the currently active set of MetricBundles. 

197 """ 

198 if self.verbose: 

199 if constraint == '': 

200 print("Querying database with no constraint.") 

201 else: 

202 print("Querying database with constraint %s" % (constraint)) 

203 # Note that we do NOT run the stackers at this point (this must be done in each 'compatible' group). 

204 if self.dbTable != 'Summary': 

205 distinctExpMJD = False 

206 groupBy = None 

207 else: 

208 distinctExpMJD = True 

209 groupBy = 'expMJD' 

210 self.simData = utils.getSimData(self.dbObj, constraint, self.dbCols, 

211 tableName=self.dbTable, distinctExpMJD=distinctExpMJD, 

212 groupBy=groupBy) 

213 

214 if self.verbose: 

215 print("Found %i visits" % (self.simData.size)) 

216 

217 # Query for the fieldData if we need it for the opsimFieldSlicer. 

218 needFields = [b.slicer.needsFields for b in self.currentBundleDict.values()] 

219 if True in needFields: 

220 self.fieldData = utils.getFieldData(self.dbObj, constraint) 

221 else: 

222 self.fieldData = None 

223 

224 def runAll(self, clearMemory=False, plotNow=False, plotKwargs=None): 

225 """Runs all the metricBundles in the metricBundleGroup, over all constraints. 

226 

227 Calculates metric values, then runs reduce functions and summary statistics for 

228 all MetricBundles. 

229 

230 Parameters 

231 ---------- 

232 clearMemory : bool, opt 

233 If True, deletes metric values from memory after running each constraint group. 

234 plotNow : bool, opt 

235 If True, plots the metric values immediately after calculation. 

236 plotKwargs : bool, opt 

237 kwargs to pass to plotCurrent. 

238 """ 

239 for constraint in self.constraints: 

240 # Set the 'currentBundleDict' which is a dictionary of the metricBundles which match this 

241 # constraint. 

242 self.setCurrent(constraint) 

243 self.runCurrent(constraint, clearMemory=clearMemory, 

244 plotNow=plotNow, plotKwargs=plotKwargs) 

245 

246 def setCurrent(self, constraint): 

247 """Utility to set the currentBundleDict (i.e. a set of metricBundles with the same SQL constraint). 

248 

249 Parameters 

250 ---------- 

251 constraint : str 

252 The subset of MetricBundles with metricBundle.constraint == constraint will be 

253 included in a subset identified as the currentBundleDict. 

254 These are the active metrics to be calculated and plotted, etc. 

255 """ 

256 if constraint is None: 

257 constraint = '' 

258 self.currentBundleDict = {} 

259 for k, b in self.bundleDict.items(): 

260 if b.constraint == constraint: 

261 self.currentBundleDict[k] = b 

262 

263 def runCurrent(self, constraint, simData=None, clearMemory=False, plotNow=False, plotKwargs=None): 

264 """Run all the metricBundles which match this constraint in the metricBundleGroup. 

265 

266 Calculates the metric values, then runs reduce functions and summary statistics for 

267 metrics in the current set only (see self.setCurrent). 

268 

269 Parameters 

270 ---------- 

271 constraint : str 

272 constraint to use to set the currently active metrics 

273 simData : numpy.ndarray, opt 

274 If simData is not None, then this numpy structured array is used instead of querying 

275 data from the dbObj. 

276 clearMemory : bool, opt 

277 If True, metric values are deleted from memory after they are calculated (and saved to disk). 

278 plotNow : bool, opt 

279 Plot immediately after calculating metric values (instead of the usual procedure, which 

280 is to plot after metric values are calculated for all constraints). 

281 plotKwargs : kwargs, opt 

282 Plotting kwargs to pass to plotCurrent. 

283 """ 

284 # Build list of all the columns needed from the database. 

285 self.dbCols = [] 

286 for b in self.currentBundleDict.values(): 

287 self.dbCols.extend(b.dbCols) 

288 self.dbCols = list(set(self.dbCols)) 

289 

290 # Can pass simData directly (if had other method for getting data) 

291 if simData is not None: 

292 self.simData = simData 

293 

294 else: 

295 self.simData = None 

296 # Query for the data. 

297 try: 

298 self.getData(constraint) 

299 except UserWarning: 

300 warnings.warn('No data matching constraint %s' % constraint) 

301 metricsSkipped = [] 

302 for b in self.currentBundleDict.values(): 

303 metricsSkipped.append("%s : %s : %s" % (b.metric.name, b.metadata, b.slicer.slicerName)) 

304 warnings.warn(' This means skipping metrics %s' % metricsSkipped) 

305 return 

306 except ValueError: 

307 warnings.warn('One or more of the columns requested from the database was not available.' + 

308 ' Skipping constraint %s' % constraint) 

309 metricsSkipped = [] 

310 for b in self.currentBundleDict.values(): 

311 metricsSkipped.append("%s : %s : %s" % (b.metric.name, b.metadata, b.slicer.slicerName)) 

312 warnings.warn(' This means skipping metrics %s' % metricsSkipped) 

313 return 

314 

315 # Find compatible subsets of the MetricBundle dictionary, 

316 # which can be run/metrics calculated/ together. 

317 self._findCompatibleLists() 

318 

319 for compatibleList in self.compatibleLists: 

320 if self.verbose: 

321 print('Running: ', compatibleList) 

322 self._runCompatible(compatibleList) 

323 if self.verbose: 

324 print('Completed metric generation.') 

325 for key in compatibleList: 

326 self.hasRun[key] = True 

327 # Run the reduce methods. 

328 if self.verbose: 

329 print('Running reduce methods.') 

330 self.reduceCurrent() 

331 # Run the summary statistics. 

332 if self.verbose: 

333 print('Running summary statistics.') 

334 self.summaryCurrent() 

335 if self.verbose: 

336 print('Completed.') 

337 if plotNow: 

338 if plotKwargs is None: 

339 self.plotCurrent() 

340 else: 

341 self.plotCurrent(**plotKwargs) 

342 # Optionally: clear results from memory. 

343 if clearMemory: 

344 for b in self.currentBundleDict.values(): 

345 b.metricValues = None 

346 if self.verbose: 

347 print('Deleted metricValues from memory.') 

348 

349 

350 def getData(self, constraint): 

351 """Query the data from the database. 

352 

353 The currently bundleDict should generally be set before calling getData (using setCurrent). 

354 

355 Parameters 

356 ---------- 

357 constraint : str 

358 The constraint for the currently active set of MetricBundles. 

359 """ 

360 if self.verbose: 

361 if constraint == '': 

362 print("Querying database %s with no constraint for columns %s." % 

363 (self.dbTable, self.dbCols)) 

364 else: 

365 print("Querying database %s with constraint %s for columns %s" % 

366 (self.dbTable, constraint, self.dbCols)) 

367 # Note that we do NOT run the stackers at this point (this must be done in each 'compatible' group). 

368 self.simData = utils.getSimData(self.dbObj, constraint, self.dbCols, 

369 groupBy='default', tableName=self.dbTable) 

370 

371 if self.verbose: 

372 print("Found %i visits" % (self.simData.size)) 

373 

374 # Query for the fieldData if we need it for the opsimFieldSlicer. 

375 needFields = [b.slicer.needsFields for b in self.currentBundleDict.values()] 

376 if True in needFields: 

377 self.fieldData = utils.getFieldData(self.dbObj, constraint) 

378 else: 

379 self.fieldData = None 

380 

381 

382 def _runCompatible(self, compatibleList): 

383 """Runs a set of 'compatible' metricbundles in the MetricBundleGroup dictionary, 

384 identified by 'compatibleList' keys. 

385 

386 A compatible list of MetricBundles is a subset of the currentBundleDict. 

387 The currentBundleDict == set of MetricBundles with the same constraint. 

388 The compatibleBundles == set of MetricBundles with the same constraint, the same 

389 slicer, the same maps applied to the slicer, and stackers which do not clobber each other's data. 

390 

391 This is where the work of calculating the metric values is done. 

392 """ 

393 

394 if len(self.simData) == 0: 

395 return 

396 

397 # Grab a dictionary representation of this subset of the dictionary, for easier iteration. 

398 bDict = {key: self.currentBundleDict.get(key) for key in compatibleList} 

399 

400 # Find the unique stackers and maps. These are already "compatible" (as id'd by compatibleList). 

401 uniqStackers = [] 

402 allStackers = [] 

403 uniqMaps = [] 

404 allMaps = [] 

405 for b in bDict.values(): 

406 allStackers += b.stackerList 

407 allMaps += b.mapsList 

408 for s in allStackers: 

409 if s not in uniqStackers: 

410 uniqStackers.append(s) 

411 for m in allMaps: 

412 if m not in uniqMaps: 

413 uniqMaps.append(m) 

414 

415 # Run stackers. 

416 # Run dither stackers first. (this is a bit of a hack -- we should probably figure out 

417 # proper hierarchy and DAG so that stackers run in the order they need to. This will catch 90%). 

418 ditherStackers = [] 

419 for s in uniqStackers: 

420 if isinstance(s, BaseDitherStacker): 

421 ditherStackers.append(s) 

422 for stacker in ditherStackers: 

423 self.simData = stacker.run(self.simData, override=True) 

424 uniqStackers.remove(stacker) 

425 

426 for stacker in uniqStackers: 

427 # Note that stackers will clobber previously existing rows with the same name. 

428 self.simData = stacker.run(self.simData, override=True) 

429 

430 # Pull out one of the slicers to use as our 'slicer'. 

431 # This will be forced back into all of the metricBundles at the end (so that they track 

432 # the same metadata such as the slicePoints, in case the same actual object wasn't used). 

433 slicer = list(bDict.values())[0].slicer 

434 if (slicer.slicerName == 'OpsimFieldSlicer'): 

435 slicer.setupSlicer(self.simData, self.fieldData, maps=uniqMaps) 

436 else: 

437 slicer.setupSlicer(self.simData, maps=uniqMaps) 

438 # Copy the slicer (after setup) back into the individual metricBundles. 

439 if slicer.slicerName != 'HealpixSlicer' or slicer.slicerName != 'UniSlicer': 

440 for b in bDict.values(): 

441 b.slicer = slicer 

442 

443 # Set up (masked) arrays to store metric data in each metricBundle. 

444 for b in bDict.values(): 

445 b._setupMetricValues() 

446 

447 # Set up an ordered dictionary to be the cache if needed: 

448 # (Currently using OrderedDict, it might be faster to use 2 regular Dicts instead) 

449 if slicer.cacheSize > 0: 

450 cacheDict = OrderedDict() 

451 cache = True 

452 else: 

453 cache = False 

454 # Run through all slicepoints and calculate metrics. 

455 for i, slice_i in enumerate(slicer): 

456 slicedata = self.simData[slice_i['idxs']] 

457 if len(slicedata) == 0: 

458 # No data at this slicepoint. Mask data values. 

459 for b in bDict.values(): 

460 b.metricValues.mask[i] = True 

461 else: 

462 # There is data! Should we use our data cache? 

463 if cache: 

464 # Make the data idxs hashable. 

465 cacheKey = frozenset(slice_i['idxs']) 

466 # If key exists, set flag to use it, otherwise add it 

467 if cacheKey in cacheDict: 

468 useCache = True 

469 cacheVal = cacheDict[cacheKey] 

470 # Move this value to the end of the OrderedDict 

471 del cacheDict[cacheKey] 

472 cacheDict[cacheKey] = cacheVal 

473 else: 

474 cacheDict[cacheKey] = i 

475 useCache = False 

476 for b in bDict.values(): 

477 if useCache: 

478 b.metricValues.data[i] = b.metricValues.data[cacheDict[cacheKey]] 

479 else: 

480 b.metricValues.data[i] = b.metric.run(slicedata, slicePoint=slice_i['slicePoint']) 

481 # If we are above the cache size, drop the oldest element from the cache dict. 

482 if len(cacheDict) > slicer.cacheSize: 

483 del cacheDict[list(cacheDict.keys())[0]] 

484 

485 # Not using memoize, just calculate things normally 

486 else: 

487 for b in bDict.values(): 

488 b.metricValues.data[i] = b.metric.run(slicedata, slicePoint=slice_i['slicePoint']) 

489 # Mask data where metrics could not be computed (according to metric bad value). 

490 for b in bDict.values(): 

491 if b.metricValues.dtype.name == 'object': 

492 for ind, val in enumerate(b.metricValues.data): 

493 if val is b.metric.badval: 

494 b.metricValues.mask[ind] = True 

495 else: 

496 # For some reason, this doesn't work for dtype=object arrays. 

497 b.metricValues.mask = np.where(b.metricValues.data == b.metric.badval, 

498 True, b.metricValues.mask) 

499 

500 # Save data to disk as we go, although this won't keep summary values, etc. (just failsafe). 

501 if self.saveEarly: 

502 for b in bDict.values(): 

503 b.write(outDir=self.outDir, resultsDb=self.resultsDb) 

504 else: 

505 for b in bDict.values(): 

506 b.writeDb(resultsDb=self.resultsDb) 

507 

508 def reduceAll(self, updateSummaries=True): 

509 """Run the reduce methods for all metrics in bundleDict. 

510 

511 Running this method, for all MetricBundles at once, assumes that clearMemory was False. 

512 

513 Parameters 

514 ---------- 

515 updateSummaries : bool, opt 

516 If True, summary metrics are removed from the top-level (non-reduced) 

517 MetricBundle. Usually this should be True, as summary metrics are generally 

518 intended to run on the simpler data produced by reduce metrics. 

519 """ 

520 for constraint in self.constraints: 

521 self.setCurrent(constraint) 

522 self.reduceCurrent(updateSummaries=updateSummaries) 

523 

524 def reduceCurrent(self, updateSummaries=True): 

525 """Run all reduce functions for the metricbundle in the currently active set of MetricBundles. 

526 

527 Parameters 

528 ---------- 

529 updateSummaries : bool, opt 

530 If True, summary metrics are removed from the top-level (non-reduced) 

531 MetricBundle. Usually this should be True, as summary metrics are generally 

532 intended to run on the simpler data produced by reduce metrics. 

533 """ 

534 # Create a temporary dictionary to hold the reduced metricbundles. 

535 reduceBundleDict = {} 

536 for b in self.currentBundleDict.values(): 

537 # If there are no reduce functions associated with the metric, skip this metricBundle. 

538 if len(b.metric.reduceFuncs) > 0: 

539 # Apply reduce functions, creating a new metricBundle in the process (new metric values). 

540 for reduceFunc in b.metric.reduceFuncs.values(): 

541 newmetricbundle = b.reduceMetric(reduceFunc) 

542 # Add the new metricBundle to our metricBundleGroup dictionary. 

543 name = newmetricbundle.metric.name 

544 if name in self.bundleDict: 

545 name = newmetricbundle.fileRoot 

546 reduceBundleDict[name] = newmetricbundle 

547 if self.saveEarly: 

548 newmetricbundle.write(outDir=self.outDir, resultsDb=self.resultsDb) 

549 else: 

550 newmetricbundle.writeDb(resultsDb=self.resultsDb) 

551 # Remove summaryMetrics from top level metricbundle if desired. 

552 if updateSummaries: 

553 b.summaryMetrics = [] 

554 # Add the new metricBundles to the MetricBundleGroup dictionary. 

555 self.bundleDict.update(reduceBundleDict) 

556 # And add to to the currentBundleDict too, so we run as part of 'summaryCurrent'. 

557 self.currentBundleDict.update(reduceBundleDict) 

558 

559 def summaryAll(self): 

560 """Run the summary statistics for all metrics in bundleDict. 

561 

562 Calculating all summary statistics, for all MetricBundles, at this 

563 point assumes that clearMemory was False. 

564 """ 

565 for constraint in self.constraints: 

566 self.setCurrent(constraint) 

567 self.summaryCurrent() 

568 

569 def summaryCurrent(self): 

570 """Run summary statistics on all the metricBundles in the currently active set of MetricBundles. 

571 """ 

572 for b in self.currentBundleDict.values(): 

573 b.computeSummaryStats(self.resultsDb) 

574 

575 def plotAll(self, savefig=True, outfileSuffix=None, figformat='pdf', dpi=600, trimWhitespace=True, 

576 thumbnail=True, closefigs=True): 

577 """Generate all the plots for all the metricBundles in bundleDict. 

578 

579 Generating all ploots, for all MetricBundles, at this point, assumes that 

580 clearMemory was False. 

581 

582 Parameters 

583 ---------- 

584 savefig : bool, opt 

585 If True, save figures to disk, to self.outDir directory. 

586 outfileSuffix : bool, opt 

587 Append outfileSuffix to the end of every plot file generated. Useful for generating 

588 sequential series of images for movies. 

589 figformat : str, opt 

590 Matplotlib figure format to use to save to disk. Default pdf. 

591 dpi : int, opt 

592 DPI for matplotlib figure. Default 600. 

593 trimWhitespace : bool, opt 

594 If True, trim additional whitespace from final figures. Default True. 

595 thumbnail : bool, opt 

596 If True, save a small thumbnail jpg version of the output file to disk as well. 

597 This is useful for showMaf web pages. Default True. 

598 closefigs : bool, opt 

599 Close the matplotlib figures after they are saved to disk. If many figures are 

600 generated, closing the figures saves significant memory. Default True. 

601 """ 

602 for constraint in self.constraints: 

603 if self.verbose: 

604 print('Plotting figures with "%s" constraint now.' % (constraint)) 

605 

606 self.setCurrent(constraint) 

607 self.plotCurrent(savefig=savefig, outfileSuffix=outfileSuffix, figformat=figformat, dpi=dpi, 

608 trimWhitespace=trimWhitespace, thumbnail=thumbnail, closefigs=closefigs) 

609 

610 def plotCurrent(self, savefig=True, outfileSuffix=None, figformat='pdf', dpi=600, trimWhitespace=True, 

611 thumbnail=True, closefigs=True): 

612 """Generate the plots for the currently active set of MetricBundles. 

613 

614 Parameters 

615 ---------- 

616 savefig : bool, opt 

617 If True, save figures to disk, to self.outDir directory. 

618 outfileSuffix : str, opt 

619 Append outfileSuffix to the end of every plot file generated. Useful for generating 

620 sequential series of images for movies. 

621 figformat : str, opt 

622 Matplotlib figure format to use to save to disk. Default pdf. 

623 dpi : int, opt 

624 DPI for matplotlib figure. Default 600. 

625 trimWhitespace : bool, opt 

626 If True, trim additional whitespace from final figures. Default True. 

627 thumbnail : bool, opt 

628 If True, save a small thumbnail jpg version of the output file to disk as well. 

629 This is useful for showMaf web pages. Default True. 

630 closefigs : bool, opt 

631 Close the matplotlib figures after they are saved to disk. If many figures are 

632 generated, closing the figures saves significant memory. Default True. 

633 """ 

634 plotHandler = PlotHandler(outDir=self.outDir, resultsDb=self.resultsDb, 

635 savefig=savefig, figformat=figformat, dpi=dpi, 

636 trimWhitespace=trimWhitespace, thumbnail=thumbnail) 

637 

638 for b in self.currentBundleDict.values(): 

639 try: 

640 b.plot(plotHandler=plotHandler, outfileSuffix=outfileSuffix, savefig=savefig) 

641 except ValueError as ve: 

642 message = 'Plotting failed for metricBundle %s.' % (b.fileRoot) 

643 message += ' Error message: %s' % (ve) 

644 warnings.warn(message) 

645 if closefigs: 

646 plt.close('all') 

647 if self.verbose: 

648 print('Plotting complete.') 

649 

650 def writeAll(self): 

651 """Save all the MetricBundles to disk. 

652 

653 Saving all MetricBundles to disk at this point assumes that clearMemory was False. 

654 """ 

655 for constraint in self.constraints: 

656 self.setCurrent(constraint) 

657 self.writeCurrent() 

658 

659 def writeCurrent(self): 

660 """Save all the MetricBundles in the currently active set to disk. 

661 """ 

662 if self.verbose: 

663 if self.saveEarly: 

664 print('Re-saving metric bundles.') 

665 else: 

666 print('Saving metric bundles.') 

667 for b in self.currentBundleDict.values(): 

668 b.write(outDir=self.outDir, resultsDb=self.resultsDb) 

669 

670 def readAll(self): 

671 """Attempt to read all MetricBundles from disk. 

672 

673 You must set the metrics/slicer/constraint/runName for a metricBundle appropriately; 

674 then this method will search for files in the location self.outDir/metricBundle.fileRoot. 

675 Reads all the files associated with all metricbundles in self.bundleDict. 

676 """ 

677 reduceBundleDict = {} 

678 removeBundles = [] 

679 for b in self.bundleDict: 

680 bundle = self.bundleDict[b] 

681 filename = os.path.join(self.outDir, bundle.fileRoot + '.npz') 

682 try: 

683 # Create a temporary metricBundle to read the data into. 

684 # (we don't use b directly, as this overrides plotDict/etc). 

685 tmpBundle = createEmptyMetricBundle() 

686 tmpBundle.read(filename) 

687 # Copy the tmpBundle metricValues into bundle. 

688 bundle.metricValues = tmpBundle.metricValues 

689 # And copy the slicer into b, to get slicePoints. 

690 bundle.slicer = tmpBundle.slicer 

691 if self.verbose: 

692 print('Read %s from disk.' % (bundle.fileRoot)) 

693 except IOError: 

694 warnings.warn('Warning: file %s not found, bundle not restored.' % filename) 

695 removeBundles.append(b) 

696 

697 # Look to see if this is a complex metric, with associated 'reduce' functions, 

698 # and read those in too. 

699 if len(bundle.metric.reduceFuncs) > 0: 

700 origMetricName = bundle.metric.name 

701 for reduceFunc in bundle.metric.reduceFuncs.values(): 

702 reduceName = origMetricName + '_' + reduceFunc.__name__.replace('reduce', '') 

703 # Borrow the fileRoot in b (we'll reset it appropriately afterwards). 

704 bundle.metric.name = reduceName 

705 bundle._buildFileRoot() 

706 filename = os.path.join(self.outDir, bundle.fileRoot + '.npz') 

707 tmpBundle = createEmptyMetricBundle() 

708 try: 

709 tmpBundle.read(filename) 

710 # This won't necessarily recreate the plotDict and displayDict exactly 

711 # as they would have been made if you calculated the reduce metric from scratch. 

712 # Perhaps update these metric reduce dictionaries after reading them in? 

713 newmetricBundle = MetricBundle(metric=bundle.metric, slicer=bundle.slicer, 

714 constraint=bundle.constraint, 

715 stackerList=bundle.stackerList, runName=bundle.runName, 

716 metadata=bundle.metadata, 

717 plotDict=bundle.plotDict, 

718 displayDict=bundle.displayDict, 

719 summaryMetrics=bundle.summaryMetrics, 

720 mapsList=bundle.mapsList, 

721 fileRoot=bundle.fileRoot, plotFuncs=bundle.plotFuncs) 

722 newmetricBundle.metric.name = reduceName 

723 newmetricBundle.metricValues = ma.copy(tmpBundle.metricValues) 

724 # Add the new metricBundle to our metricBundleGroup dictionary. 

725 name = newmetricBundle.metric.name 

726 if name in self.bundleDict: 

727 name = newmetricBundle.fileRoot 

728 reduceBundleDict[name] = newmetricBundle 

729 if self.verbose: 

730 print('Read %s from disk.' % (newmetricBundle.fileRoot)) 

731 except IOError: 

732 warnings.warn('Warning: file %s not found, bundle not restored ("reduce" metric).' 

733 % filename) 

734 

735 # Remove summaryMetrics from top level metricbundle. 

736 bundle.summaryMetrics = [] 

737 # Update parent MetricBundle name. 

738 bundle.metric.name = origMetricName 

739 bundle._buildFileRoot() 

740 

741 # Add the reduce bundles into the bundleDict. 

742 self.bundleDict.update(reduceBundleDict) 

743 # And remove the bundles which were not found on disk, so we don't try to make (blank) plots. 

744 for b in removeBundles: 

745 del self.bundleDict[b] 

746