Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from __future__ import print_function 

2from builtins import object 

3import os 

4import numpy as np 

5import numpy.ma as ma 

6import matplotlib.pyplot as plt 

7from collections import OrderedDict 

8 

9import lsst.sims.maf.db as db 

10import lsst.sims.maf.utils as utils 

11from lsst.sims.maf.plots import PlotHandler 

12import lsst.sims.maf.maps as maps 

13from lsst.sims.maf.stackers import BaseDitherStacker 

14from .metricBundle import MetricBundle, createEmptyMetricBundle 

15import warnings 

16 

17__all__ = ['makeBundlesDictFromList', 'MetricBundleGroup'] 

18 

19 

20def makeBundlesDictFromList(bundleList): 

21 """Utility to convert a list of MetricBundles into a dictionary, keyed by the fileRoot names. 

22 

23 Raises an exception if the fileroot duplicates another metricBundle. 

24 (Note this should alert to potential cases of filename duplication). 

25 

26 Parameters 

27 ---------- 

28 bundleList : list of MetricBundles 

29 """ 

30 bDict = {} 

31 for b in bundleList: 

32 if b.fileRoot in bDict: 

33 raise NameError('More than one metricBundle is using the same fileroot, %s' % (b.fileRoot)) 

34 bDict[b.fileRoot] = b 

35 return bDict 

36 

37 

38class MetricBundleGroup(object): 

39 """The MetricBundleGroup exists to calculate the metric values for a group of 

40 MetricBundles. 

41 

42 The MetricBundleGroup will query data from a single database table (for multiple 

43 constraints), use that data to calculate metric values for multiple slicers, 

44 and calculate summary statistics and generate plots for all metrics included in 

45 the dictionary passed to the MetricBundleGroup. 

46 

47 We calculate the metric values here, rather than in the individual MetricBundles, 

48 because it is much more efficient to step through a slicer once (and calculate all 

49 the relevant metric values at each point) than it is to repeat this process multiple times. 

50 

51 The MetricBundleGroup also determines how to efficiently group the MetricBundles 

52 to reduce the number of sql queries of the database, grabbing larger chunks of data at once. 

53 

54 Parameters 

55 ---------- 

56 bundleDict : dict or list of MetricBundles 

57 Individual MetricBundles should be placed into a dictionary, and then passed to 

58 the MetricBundleGroup. The dictionary keys can then be used to identify MetricBundles 

59 if needed -- and to identify new MetricBundles which could be created if 'reduce' 

60 functions are run on a particular MetricBundle. 

61 A bundleDict can be conveniently created from a list of MetricBundles using 

62 makeBundlesDictFromList (done automatically if a list is passed in) 

63 dbObj : Database 

64 The database object (typically an :class:`OpsimDatabase`) connected to the data to be used to 

65 calculate metrics. 

66 Advanced use: It is possible to set this to None, in which case data should be passed 

67 directly to the runCurrent method (and runAll should not be used). 

68 outDir : str, opt 

69 Directory to save the metric results. Default is the current directory. 

70 resultsDb : ResultsDb, opt 

71 A results database. If not specified, one will be created in the outDir. 

72 This database saves information about the metrics calculated, including their summary statistics. 

73 verbose : bool, opt 

74 Flag to turn on/off verbose feedback. 

75 saveEarly : bool, opt 

76 If True, metric values will be saved immediately after they are first calculated (to prevent 

77 data loss) as well as after summary statistics are calculated. 

78 If False, metric values will only be saved after summary statistics are calculated. 

79 dbTable : str, opt 

80 The name of the table in the dbObj to query for data. 

81 """ 

82 def __init__(self, bundleDict, dbObj, outDir='.', resultsDb=None, verbose=True, 

83 saveEarly=True, dbTable=None): 

84 """Set up the MetricBundleGroup. 

85 """ 

86 if type(bundleDict) is list: 

87 bundleDict = makeBundlesDictFromList(bundleDict) 

88 # Print occasional messages to screen. 

89 self.verbose = verbose 

90 # Save metric results as soon as possible (in case of crash). 

91 self.saveEarly = saveEarly 

92 # Check for output directory, create it if needed. 

93 self.outDir = outDir 

94 if not os.path.isdir(self.outDir): 

95 os.makedirs(self.outDir) 

96 

97 # Do some type checking on the MetricBundle dictionary. 

98 if not isinstance(bundleDict, dict): 

99 raise ValueError('bundleDict should be a dictionary containing MetricBundle objects.') 

100 for b in bundleDict.values(): 

101 if not isinstance(b, MetricBundle): 

102 raise ValueError('bundleDict should contain only MetricBundle objects.') 

103 # Identify the series of constraints. 

104 self.constraints = list(set([b.constraint for b in bundleDict.values()])) 

105 # Set the bundleDict (all bundles, with all constraints) 

106 self.bundleDict = bundleDict 

107 

108 # Check the dbObj. 

109 if not isinstance(dbObj, db.Database): 

110 warnings.warn('Warning: dbObj should be an instantiated Database (or child) object.') 

111 self.dbObj = dbObj 

112 # Set the table we're going to be querying. 

113 self.dbTable = dbTable 

114 if self.dbTable is None and self.dbObj is not None: 

115 self.dbTable = self.dbObj.defaultTable 

116 

117 # Check the resultsDb (optional). 

118 if resultsDb is not None: 

119 if not isinstance(resultsDb, db.ResultsDb): 

120 raise ValueError('resultsDb should be an ResultsDb object') 

121 self.resultsDb = resultsDb 

122 

123 # Dict to keep track of what's been run: 

124 self.hasRun = {} 

125 for bk in bundleDict: 

126 self.hasRun[bk] = False 

127 

128 def _checkCompatible(self, metricBundle1, metricBundle2): 

129 """Check if two MetricBundles are "compatible". 

130 Compatible indicates that the sql constraints, the slicers, and the maps are the same, and 

131 that the stackers do not interfere with each other 

132 (i.e. are not trying to set the same column in different ways). 

133 Returns True if the MetricBundles are compatible, False if not. 

134 

135 Parameters 

136 ---------- 

137 metricBundle1 : MetricBundle 

138 metricBundle2 : MetricBundle 

139 

140 Returns 

141 ------- 

142 bool 

143 """ 

144 if metricBundle1.constraint != metricBundle2.constraint: 

145 return False 

146 if metricBundle1.slicer != metricBundle2.slicer: 

147 return False 

148 if metricBundle1.mapsList.sort() != metricBundle2.mapsList.sort(): 

149 return False 

150 for stacker in metricBundle1.stackerList: 

151 for stacker2 in metricBundle2.stackerList: 

152 # If the stackers have different names, that's OK, and if they are identical, that's ok. 

153 if (stacker.__class__.__name__ == stacker2.__class__.__name__) & (stacker != stacker2): 

154 return False 

155 # But if we got this far, everything matches. 

156 return True 

157 

158 def _findCompatibleLists(self): 

159 """Find sets of compatible metricBundles from the currentBundleDict. 

160 """ 

161 # CompatibleLists stores a list of lists; 

162 # each (nested) list contains the bundleDict _keys_ of a compatible set of metricBundles. 

163 # 

164 compatibleLists = [] 

165 for k, b in self.currentBundleDict.items(): 

166 foundCompatible = False 

167 for compatibleList in compatibleLists: 

168 comparisonMetricBundleKey = compatibleList[0] 

169 compatible = self._checkCompatible(self.bundleDict[comparisonMetricBundleKey], b) 

170 if compatible: 

171 # Must compare all metricBundles in each subset (if they are a potential match), 

172 # as the stackers could be different (and one could be incompatible, 

173 # not necessarily the first) 

174 for comparisonMetricBundleKey in compatibleList[1:]: 

175 compatible = self._checkCompatible(self.bundleDict[comparisonMetricBundleKey], b) 

176 if not compatible: 

177 # If we find one which is not compatible, stop and go on to the 

178 # next subset list. 

179 break 

180 # Otherwise, we reached the end of the subset and they were all compatible. 

181 foundCompatible = True 

182 compatibleList.append(k) 

183 if not foundCompatible: 

184 # Didn't find a pre-existing compatible set; make a new one. 

185 compatibleLists.append([k, ]) 

186 self.compatibleLists = compatibleLists 

187 

188 def getData(self, constraint): 

189 """Query the data from the database. 

190 

191 The currently bundleDict should generally be set before calling getData (using setCurrent). 

192 

193 Parameters 

194 ---------- 

195 constraint : str 

196 The constraint for the currently active set of MetricBundles. 

197 """ 

198 if self.verbose: 

199 if constraint == '': 

200 print("Querying database with no constraint.") 

201 else: 

202 print("Querying database with constraint %s" % (constraint)) 

203 # Note that we do NOT run the stackers at this point (this must be done in each 'compatible' group). 

204 if self.dbTable != 'Summary': 

205 distinctExpMJD = False 

206 groupBy = None 

207 else: 

208 distinctExpMJD = True 

209 groupBy = 'expMJD' 

210 self.simData = utils.getSimData(self.dbObj, constraint, self.dbCols, 

211 tableName=self.dbTable, distinctExpMJD=distinctExpMJD, 

212 groupBy=groupBy) 

213 

214 if self.verbose: 

215 print("Found %i visits" % (self.simData.size)) 

216 

217 # Query for the fieldData if we need it for the opsimFieldSlicer. 

218 needFields = [b.slicer.needsFields for b in self.currentBundleDict.values()] 

219 if True in needFields: 

220 self.fieldData = utils.getFieldData(self.dbObj, constraint) 

221 else: 

222 self.fieldData = None 

223 

224 def runAll(self, clearMemory=False, plotNow=False, plotKwargs=None): 

225 """Runs all the metricBundles in the metricBundleGroup, over all constraints. 

226 

227 Calculates metric values, then runs reduce functions and summary statistics for 

228 all MetricBundles. 

229 

230 Parameters 

231 ---------- 

232 clearMemory : bool, opt 

233 If True, deletes metric values from memory after running each constraint group. 

234 plotNow : bool, opt 

235 If True, plots the metric values immediately after calculation. 

236 plotKwargs : bool, opt 

237 kwargs to pass to plotCurrent. 

238 """ 

239 for constraint in self.constraints: 

240 # Set the 'currentBundleDict' which is a dictionary of the metricBundles which match this 

241 # constraint. 

242 self.runCurrent(constraint, clearMemory=clearMemory, 

243 plotNow=plotNow, plotKwargs=plotKwargs) 

244 

245 def setCurrent(self, constraint): 

246 """Utility to set the currentBundleDict (i.e. a set of metricBundles with the same SQL constraint). 

247 

248 Parameters 

249 ---------- 

250 constraint : str 

251 The subset of MetricBundles with metricBundle.constraint == constraint will be 

252 included in a subset identified as the currentBundleDict. 

253 These are the active metrics to be calculated and plotted, etc. 

254 """ 

255 if constraint is None: 

256 constraint = '' 

257 self.currentBundleDict = {} 

258 for k, b in self.bundleDict.items(): 

259 if b.constraint == constraint: 

260 self.currentBundleDict[k] = b 

261 

262 def runCurrent(self, constraint, simData=None, clearMemory=False, plotNow=False, plotKwargs=None): 

263 """Run all the metricBundles which match this constraint in the metricBundleGroup. 

264 

265 Calculates the metric values, then runs reduce functions and summary statistics for 

266 metrics in the current set only (see self.setCurrent). 

267 

268 Parameters 

269 ---------- 

270 constraint : str 

271 constraint to use to set the currently active metrics 

272 simData : numpy.ndarray, opt 

273 If simData is not None, then this numpy structured array is used instead of querying 

274 data from the dbObj. 

275 clearMemory : bool, opt 

276 If True, metric values are deleted from memory after they are calculated (and saved to disk). 

277 plotNow : bool, opt 

278 Plot immediately after calculating metric values (instead of the usual procedure, which 

279 is to plot after metric values are calculated for all constraints). 

280 plotKwargs : kwargs, opt 

281 Plotting kwargs to pass to plotCurrent. 

282 """ 

283 self.setCurrent(constraint) 

284 

285 # Build list of all the columns needed from the database. 

286 self.dbCols = [] 

287 for b in self.currentBundleDict.values(): 

288 self.dbCols.extend(b.dbCols) 

289 self.dbCols = list(set(self.dbCols)) 

290 

291 # Can pass simData directly (if had other method for getting data) 

292 if simData is not None: 

293 self.simData = simData 

294 

295 else: 

296 self.simData = None 

297 # Query for the data. 

298 try: 

299 self.getData(constraint) 

300 except UserWarning: 

301 warnings.warn('No data matching constraint %s' % constraint) 

302 metricsSkipped = [] 

303 for b in self.currentBundleDict.values(): 

304 metricsSkipped.append("%s : %s : %s" % (b.metric.name, b.metadata, b.slicer.slicerName)) 

305 warnings.warn(' This means skipping metrics %s' % metricsSkipped) 

306 return 

307 except ValueError: 

308 warnings.warn('One or more of the columns requested from the database was not available.' + 

309 ' Skipping constraint %s' % constraint) 

310 metricsSkipped = [] 

311 for b in self.currentBundleDict.values(): 

312 metricsSkipped.append("%s : %s : %s" % (b.metric.name, b.metadata, b.slicer.slicerName)) 

313 warnings.warn(' This means skipping metrics %s' % metricsSkipped) 

314 return 

315 

316 # Find compatible subsets of the MetricBundle dictionary, 

317 # which can be run/metrics calculated/ together. 

318 self._findCompatibleLists() 

319 

320 for compatibleList in self.compatibleLists: 

321 if self.verbose: 

322 print('Running: ', compatibleList) 

323 self._runCompatible(compatibleList) 

324 if self.verbose: 

325 print('Completed metric generation.') 

326 for key in compatibleList: 

327 self.hasRun[key] = True 

328 # Run the reduce methods. 

329 if self.verbose: 

330 print('Running reduce methods.') 

331 self.reduceCurrent() 

332 # Run the summary statistics. 

333 if self.verbose: 

334 print('Running summary statistics.') 

335 self.summaryCurrent() 

336 if self.verbose: 

337 print('Completed.') 

338 if plotNow: 

339 if plotKwargs is None: 

340 self.plotCurrent() 

341 else: 

342 self.plotCurrent(**plotKwargs) 

343 # Optionally: clear results from memory. 

344 if clearMemory: 

345 for b in self.currentBundleDict.values(): 

346 b.metricValues = None 

347 if self.verbose: 

348 print('Deleted metricValues from memory.') 

349 

350 

351 def getData(self, constraint): 

352 """Query the data from the database. 

353 

354 The currently bundleDict should generally be set before calling getData (using setCurrent). 

355 

356 Parameters 

357 ---------- 

358 constraint : str 

359 The constraint for the currently active set of MetricBundles. 

360 """ 

361 if self.verbose: 

362 if constraint == '': 

363 print("Querying database %s with no constraint for columns %s." % 

364 (self.dbTable, self.dbCols)) 

365 else: 

366 print("Querying database %s with constraint %s for columns %s" % 

367 (self.dbTable, constraint, self.dbCols)) 

368 # Note that we do NOT run the stackers at this point (this must be done in each 'compatible' group). 

369 self.simData = utils.getSimData(self.dbObj, constraint, self.dbCols, 

370 groupBy='default', tableName=self.dbTable) 

371 

372 if self.verbose: 

373 print("Found %i visits" % (self.simData.size)) 

374 

375 # Query for the fieldData if we need it for the opsimFieldSlicer. 

376 needFields = [b.slicer.needsFields for b in self.currentBundleDict.values()] 

377 if True in needFields: 

378 self.fieldData = utils.getFieldData(self.dbObj, constraint) 

379 else: 

380 self.fieldData = None 

381 

382 

383 def _runCompatible(self, compatibleList): 

384 """Runs a set of 'compatible' metricbundles in the MetricBundleGroup dictionary, 

385 identified by 'compatibleList' keys. 

386 

387 A compatible list of MetricBundles is a subset of the currentBundleDict. 

388 The currentBundleDict == set of MetricBundles with the same constraint. 

389 The compatibleBundles == set of MetricBundles with the same constraint, the same 

390 slicer, the same maps applied to the slicer, and stackers which do not clobber each other's data. 

391 

392 This is where the work of calculating the metric values is done. 

393 """ 

394 

395 if len(self.simData) == 0: 

396 return 

397 

398 # Grab a dictionary representation of this subset of the dictionary, for easier iteration. 

399 bDict = {key: self.currentBundleDict.get(key) for key in compatibleList} 

400 

401 # Find the unique stackers and maps. These are already "compatible" (as id'd by compatibleList). 

402 uniqStackers = [] 

403 allStackers = [] 

404 uniqMaps = [] 

405 allMaps = [] 

406 for b in bDict.values(): 

407 allStackers += b.stackerList 

408 allMaps += b.mapsList 

409 for s in allStackers: 

410 if s not in uniqStackers: 

411 uniqStackers.append(s) 

412 for m in allMaps: 

413 if m not in uniqMaps: 

414 uniqMaps.append(m) 

415 

416 # Run stackers. 

417 # Run dither stackers first. (this is a bit of a hack -- we should probably figure out 

418 # proper hierarchy and DAG so that stackers run in the order they need to. This will catch 90%). 

419 ditherStackers = [] 

420 for s in uniqStackers: 

421 if isinstance(s, BaseDitherStacker): 

422 ditherStackers.append(s) 

423 for stacker in ditherStackers: 

424 self.simData = stacker.run(self.simData, override=True) 

425 uniqStackers.remove(stacker) 

426 

427 for stacker in uniqStackers: 

428 # Note that stackers will clobber previously existing rows with the same name. 

429 self.simData = stacker.run(self.simData, override=True) 

430 

431 # Pull out one of the slicers to use as our 'slicer'. 

432 # This will be forced back into all of the metricBundles at the end (so that they track 

433 # the same metadata such as the slicePoints, in case the same actual object wasn't used). 

434 slicer = list(bDict.values())[0].slicer 

435 if (slicer.slicerName == 'OpsimFieldSlicer'): 

436 slicer.setupSlicer(self.simData, self.fieldData, maps=uniqMaps) 

437 else: 

438 slicer.setupSlicer(self.simData, maps=uniqMaps) 

439 # Copy the slicer (after setup) back into the individual metricBundles. 

440 if slicer.slicerName != 'HealpixSlicer' or slicer.slicerName != 'UniSlicer': 

441 for b in bDict.values(): 

442 b.slicer = slicer 

443 

444 # Set up (masked) arrays to store metric data in each metricBundle. 

445 for b in bDict.values(): 

446 b._setupMetricValues() 

447 

448 # Set up an ordered dictionary to be the cache if needed: 

449 # (Currently using OrderedDict, it might be faster to use 2 regular Dicts instead) 

450 if slicer.cacheSize > 0: 

451 cacheDict = OrderedDict() 

452 cache = True 

453 else: 

454 cache = False 

455 # Run through all slicepoints and calculate metrics. 

456 for i, slice_i in enumerate(slicer): 

457 slicedata = self.simData[slice_i['idxs']] 

458 if len(slicedata) == 0: 

459 # No data at this slicepoint. Mask data values. 

460 for b in bDict.values(): 

461 b.metricValues.mask[i] = True 

462 else: 

463 # There is data! Should we use our data cache? 

464 if cache: 

465 # Make the data idxs hashable. 

466 cacheKey = frozenset(slice_i['idxs']) 

467 # If key exists, set flag to use it, otherwise add it 

468 if cacheKey in cacheDict: 

469 useCache = True 

470 cacheVal = cacheDict[cacheKey] 

471 # Move this value to the end of the OrderedDict 

472 del cacheDict[cacheKey] 

473 cacheDict[cacheKey] = cacheVal 

474 else: 

475 cacheDict[cacheKey] = i 

476 useCache = False 

477 for b in bDict.values(): 

478 if useCache: 

479 b.metricValues.data[i] = b.metricValues.data[cacheDict[cacheKey]] 

480 else: 

481 b.metricValues.data[i] = b.metric.run(slicedata, slicePoint=slice_i['slicePoint']) 

482 # If we are above the cache size, drop the oldest element from the cache dict. 

483 if len(cacheDict) > slicer.cacheSize: 

484 del cacheDict[list(cacheDict.keys())[0]] 

485 

486 # Not using memoize, just calculate things normally 

487 else: 

488 for b in bDict.values(): 

489 b.metricValues.data[i] = b.metric.run(slicedata, slicePoint=slice_i['slicePoint']) 

490 # Mask data where metrics could not be computed (according to metric bad value). 

491 for b in bDict.values(): 

492 if b.metricValues.dtype.name == 'object': 

493 for ind, val in enumerate(b.metricValues.data): 

494 if val is b.metric.badval: 

495 b.metricValues.mask[ind] = True 

496 else: 

497 # For some reason, this doesn't work for dtype=object arrays. 

498 b.metricValues.mask = np.where(b.metricValues.data == b.metric.badval, 

499 True, b.metricValues.mask) 

500 

501 # Save data to disk as we go, although this won't keep summary values, etc. (just failsafe). 

502 if self.saveEarly: 

503 for b in bDict.values(): 

504 b.write(outDir=self.outDir, resultsDb=self.resultsDb) 

505 else: 

506 for b in bDict.values(): 

507 b.writeDb(resultsDb=self.resultsDb) 

508 

509 def reduceAll(self, updateSummaries=True): 

510 """Run the reduce methods for all metrics in bundleDict. 

511 

512 Running this method, for all MetricBundles at once, assumes that clearMemory was False. 

513 

514 Parameters 

515 ---------- 

516 updateSummaries : bool, opt 

517 If True, summary metrics are removed from the top-level (non-reduced) 

518 MetricBundle. Usually this should be True, as summary metrics are generally 

519 intended to run on the simpler data produced by reduce metrics. 

520 """ 

521 for constraint in self.constraints: 

522 self.setCurrent(constraint) 

523 self.reduceCurrent(updateSummaries=updateSummaries) 

524 

525 def reduceCurrent(self, updateSummaries=True): 

526 """Run all reduce functions for the metricbundle in the currently active set of MetricBundles. 

527 

528 Parameters 

529 ---------- 

530 updateSummaries : bool, opt 

531 If True, summary metrics are removed from the top-level (non-reduced) 

532 MetricBundle. Usually this should be True, as summary metrics are generally 

533 intended to run on the simpler data produced by reduce metrics. 

534 """ 

535 # Create a temporary dictionary to hold the reduced metricbundles. 

536 reduceBundleDict = {} 

537 for b in self.currentBundleDict.values(): 

538 # If there are no reduce functions associated with the metric, skip this metricBundle. 

539 if len(b.metric.reduceFuncs) > 0: 

540 # Apply reduce functions, creating a new metricBundle in the process (new metric values). 

541 for reduceFunc in b.metric.reduceFuncs.values(): 

542 newmetricbundle = b.reduceMetric(reduceFunc) 

543 # Add the new metricBundle to our metricBundleGroup dictionary. 

544 name = newmetricbundle.metric.name 

545 if name in self.bundleDict: 

546 name = newmetricbundle.fileRoot 

547 reduceBundleDict[name] = newmetricbundle 

548 if self.saveEarly: 

549 newmetricbundle.write(outDir=self.outDir, resultsDb=self.resultsDb) 

550 else: 

551 newmetricbundle.writeDb(resultsDb=self.resultsDb) 

552 # Remove summaryMetrics from top level metricbundle if desired. 

553 if updateSummaries: 

554 b.summaryMetrics = [] 

555 # Add the new metricBundles to the MetricBundleGroup dictionary. 

556 self.bundleDict.update(reduceBundleDict) 

557 # And add to to the currentBundleDict too, so we run as part of 'summaryCurrent'. 

558 self.currentBundleDict.update(reduceBundleDict) 

559 

560 def summaryAll(self): 

561 """Run the summary statistics for all metrics in bundleDict. 

562 

563 Calculating all summary statistics, for all MetricBundles, at this 

564 point assumes that clearMemory was False. 

565 """ 

566 for constraint in self.constraints: 

567 self.setCurrent(constraint) 

568 self.summaryCurrent() 

569 

570 def summaryCurrent(self): 

571 """Run summary statistics on all the metricBundles in the currently active set of MetricBundles. 

572 """ 

573 for b in self.currentBundleDict.values(): 

574 b.computeSummaryStats(self.resultsDb) 

575 

576 def plotAll(self, savefig=True, outfileSuffix=None, figformat='pdf', dpi=600, trimWhitespace=True, 

577 thumbnail=True, closefigs=True): 

578 """Generate all the plots for all the metricBundles in bundleDict. 

579 

580 Generating all ploots, for all MetricBundles, at this point, assumes that 

581 clearMemory was False. 

582 

583 Parameters 

584 ---------- 

585 savefig : bool, opt 

586 If True, save figures to disk, to self.outDir directory. 

587 outfileSuffix : bool, opt 

588 Append outfileSuffix to the end of every plot file generated. Useful for generating 

589 sequential series of images for movies. 

590 figformat : str, opt 

591 Matplotlib figure format to use to save to disk. Default pdf. 

592 dpi : int, opt 

593 DPI for matplotlib figure. Default 600. 

594 trimWhitespace : bool, opt 

595 If True, trim additional whitespace from final figures. Default True. 

596 thumbnail : bool, opt 

597 If True, save a small thumbnail jpg version of the output file to disk as well. 

598 This is useful for showMaf web pages. Default True. 

599 closefigs : bool, opt 

600 Close the matplotlib figures after they are saved to disk. If many figures are 

601 generated, closing the figures saves significant memory. Default True. 

602 """ 

603 for constraint in self.constraints: 

604 if self.verbose: 

605 print('Plotting figures with "%s" constraint now.' % (constraint)) 

606 

607 self.setCurrent(constraint) 

608 self.plotCurrent(savefig=savefig, outfileSuffix=outfileSuffix, figformat=figformat, dpi=dpi, 

609 trimWhitespace=trimWhitespace, thumbnail=thumbnail, closefigs=closefigs) 

610 

611 def plotCurrent(self, savefig=True, outfileSuffix=None, figformat='pdf', dpi=600, trimWhitespace=True, 

612 thumbnail=True, closefigs=True): 

613 """Generate the plots for the currently active set of MetricBundles. 

614 

615 Parameters 

616 ---------- 

617 savefig : bool, opt 

618 If True, save figures to disk, to self.outDir directory. 

619 outfileSuffix : str, opt 

620 Append outfileSuffix to the end of every plot file generated. Useful for generating 

621 sequential series of images for movies. 

622 figformat : str, opt 

623 Matplotlib figure format to use to save to disk. Default pdf. 

624 dpi : int, opt 

625 DPI for matplotlib figure. Default 600. 

626 trimWhitespace : bool, opt 

627 If True, trim additional whitespace from final figures. Default True. 

628 thumbnail : bool, opt 

629 If True, save a small thumbnail jpg version of the output file to disk as well. 

630 This is useful for showMaf web pages. Default True. 

631 closefigs : bool, opt 

632 Close the matplotlib figures after they are saved to disk. If many figures are 

633 generated, closing the figures saves significant memory. Default True. 

634 """ 

635 plotHandler = PlotHandler(outDir=self.outDir, resultsDb=self.resultsDb, 

636 savefig=savefig, figformat=figformat, dpi=dpi, 

637 trimWhitespace=trimWhitespace, thumbnail=thumbnail) 

638 

639 for b in self.currentBundleDict.values(): 

640 try: 

641 b.plot(plotHandler=plotHandler, outfileSuffix=outfileSuffix, savefig=savefig) 

642 except ValueError as ve: 

643 message = 'Plotting failed for metricBundle %s.' % (b.fileRoot) 

644 message += ' Error message: %s' % (ve) 

645 warnings.warn(message) 

646 if closefigs: 

647 plt.close('all') 

648 if self.verbose: 

649 print('Plotting complete.') 

650 

651 def writeAll(self): 

652 """Save all the MetricBundles to disk. 

653 

654 Saving all MetricBundles to disk at this point assumes that clearMemory was False. 

655 """ 

656 for constraint in self.constraints: 

657 self.setCurrent(constraint) 

658 self.writeCurrent() 

659 

660 def writeCurrent(self): 

661 """Save all the MetricBundles in the currently active set to disk. 

662 """ 

663 if self.verbose: 

664 if self.saveEarly: 

665 print('Re-saving metric bundles.') 

666 else: 

667 print('Saving metric bundles.') 

668 for b in self.currentBundleDict.values(): 

669 b.write(outDir=self.outDir, resultsDb=self.resultsDb) 

670 

671 def readAll(self): 

672 """Attempt to read all MetricBundles from disk. 

673 

674 You must set the metrics/slicer/constraint/runName for a metricBundle appropriately; 

675 then this method will search for files in the location self.outDir/metricBundle.fileRoot. 

676 Reads all the files associated with all metricbundles in self.bundleDict. 

677 """ 

678 reduceBundleDict = {} 

679 removeBundles = [] 

680 for b in self.bundleDict: 

681 bundle = self.bundleDict[b] 

682 filename = os.path.join(self.outDir, bundle.fileRoot + '.npz') 

683 try: 

684 # Create a temporary metricBundle to read the data into. 

685 # (we don't use b directly, as this overrides plotDict/etc). 

686 tmpBundle = createEmptyMetricBundle() 

687 tmpBundle.read(filename) 

688 # Copy the tmpBundle metricValues into bundle. 

689 bundle.metricValues = tmpBundle.metricValues 

690 # And copy the slicer into b, to get slicePoints. 

691 bundle.slicer = tmpBundle.slicer 

692 if self.verbose: 

693 print('Read %s from disk.' % (bundle.fileRoot)) 

694 except IOError: 

695 warnings.warn('Warning: file %s not found, bundle not restored.' % filename) 

696 removeBundles.append(b) 

697 

698 # Look to see if this is a complex metric, with associated 'reduce' functions, 

699 # and read those in too. 

700 if len(bundle.metric.reduceFuncs) > 0: 

701 origMetricName = bundle.metric.name 

702 for reduceFunc in bundle.metric.reduceFuncs.values(): 

703 reduceName = origMetricName + '_' + reduceFunc.__name__.replace('reduce', '') 

704 # Borrow the fileRoot in b (we'll reset it appropriately afterwards). 

705 bundle.metric.name = reduceName 

706 bundle._buildFileRoot() 

707 filename = os.path.join(self.outDir, bundle.fileRoot + '.npz') 

708 tmpBundle = createEmptyMetricBundle() 

709 try: 

710 tmpBundle.read(filename) 

711 # This won't necessarily recreate the plotDict and displayDict exactly 

712 # as they would have been made if you calculated the reduce metric from scratch. 

713 # Perhaps update these metric reduce dictionaries after reading them in? 

714 newmetricBundle = MetricBundle(metric=bundle.metric, slicer=bundle.slicer, 

715 constraint=bundle.constraint, 

716 stackerList=bundle.stackerList, runName=bundle.runName, 

717 metadata=bundle.metadata, 

718 plotDict=bundle.plotDict, 

719 displayDict=bundle.displayDict, 

720 summaryMetrics=bundle.summaryMetrics, 

721 mapsList=bundle.mapsList, 

722 fileRoot=bundle.fileRoot, plotFuncs=bundle.plotFuncs) 

723 newmetricBundle.metric.name = reduceName 

724 newmetricBundle.metricValues = ma.copy(tmpBundle.metricValues) 

725 # Add the new metricBundle to our metricBundleGroup dictionary. 

726 name = newmetricBundle.metric.name 

727 if name in self.bundleDict: 

728 name = newmetricBundle.fileRoot 

729 reduceBundleDict[name] = newmetricBundle 

730 if self.verbose: 

731 print('Read %s from disk.' % (newmetricBundle.fileRoot)) 

732 except IOError: 

733 warnings.warn('Warning: file %s not found, bundle not restored ("reduce" metric).' 

734 % filename) 

735 

736 # Remove summaryMetrics from top level metricbundle. 

737 bundle.summaryMetrics = [] 

738 # Update parent MetricBundle name. 

739 bundle.metric.name = origMetricName 

740 bundle._buildFileRoot() 

741 

742 # Add the reduce bundles into the bundleDict. 

743 self.bundleDict.update(reduceBundleDict) 

744 # And remove the bundles which were not found on disk, so we don't try to make (blank) plots. 

745 for b in removeBundles: 

746 del self.bundleDict[b] 

747