Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

from __future__ import print_function 

from builtins import str 

from builtins import object 

import os 

from sqlalchemy import create_engine, Column, Integer, String 

from sqlalchemy.engine import url 

from sqlalchemy.orm import sessionmaker 

 

from sqlalchemy.ext.declarative import declarative_base 

from sqlalchemy.exc import DatabaseError 

from lsst.daf.persistence import DbAuth 

 

Base = declarative_base() 

 

__all__ = ['TrackingDb', 'addRunToDatabase'] 

 

 

class RunRow(Base): 

""" 

Define contents and format of run list table. 

 

Table to list all available MAF results, along with their opsim run and some comment info. 

""" 

__tablename__ = "runs" 

# Define columns in metric list table. 

mafRunId = Column(Integer, primary_key=True) 

opsimGroup = Column(String) 

opsimRun = Column(String) 

opsimComment = Column(String) 

opsimVersion = Column(String) 

opsimDate = Column(String) 

dbFile = Column(String) 

mafComment = Column(String) 

mafVersion = Column(String) 

mafDate = Column(String) 

mafDir = Column(String) 

def __repr__(self): 

rstr = "<Run(mafRunId='%d', opsimGroup='%s', opsimRun='%s', opsimComment='%s', " \ 

"opsimVersion='%s', opsimDate='%s', mafComment='%s', " \ 

"mafVersion='%s', mafDate='%s', mafDir='%s', dbFile='%s'>" \ 

% (self.mafRunId, self.opsimGroup, self.opsimRun, self.opsimComment, 

self.opsimVersion, self.opsimDate, self.mafComment, self.mafVersion, self.mafDate, 

self.mafDir, self.dbFile) 

return rstr 

 

 

class TrackingDb(object): 

 

def __init__(self, database=None, driver='sqlite', host=None, port=None, 

trackingDbverbose=False): 

""" 

Instantiate the results database, creating metrics, plots and summarystats tables. 

""" 

self.verbose = trackingDbverbose 

# Connect to database 

# for sqlite, connecting to non-existent database creates it automatically 

if database is None: 

# Default is a file in the current directory. 

self.database = os.path.join(os.getcwd(), 'trackingDb_sqlite.db') 

self.driver = 'sqlite' 

else: 

self.database = database 

self.driver = driver 

self.host = host 

self.port = port 

 

if self.driver == 'sqlite': 

dbAddress = url.URL(drivername=self.driver, database=self.database) 

else: 

dbAddress = url.URL(self.driver, 

username=DbAuth.username(self.host, str(self.port)), 

password=DbAuth.password(self.host, str(self.port)), 

host=self.host, 

port=self.port, 

database=self.database) 

 

engine = create_engine(dbAddress, echo=self.verbose) 

if self.verbose: 

print('Created or connected to MAF tracking %s database at %s' %(self.driver, self.database)) 

self.Session = sessionmaker(bind=engine) 

self.session = self.Session() 

# Create the tables, if they don't already exist. 

try: 

Base.metadata.create_all(engine) 

except DatabaseError: 

raise DatabaseError("Cannot create a %s database at %s. Check directory exists." %(self.driver, self.database)) 

 

def close(self): 

self.session.close() 

 

def addRun(self, opsimGroup=None, opsimRun=None, opsimComment=None, opsimVersion=None, opsimDate=None, 

mafComment=None, mafVersion=None, mafDate=None, mafDir=None, dbFile=None, mafRunId=None): 

"""Add a run to the tracking database. 

 

Parameters 

---------- 

opsimGroup : str, opt 

Set a name to group this run with (eg. "Tier 1, 2016"). 

opsimRun : str, opt 

Set a name for the opsim run. 

opsimComment : str, opt 

Set a comment describing the opsim run. 

opsimVersion : str, opt 

Set the version of opsim. 

opsimDate : str, opt 

Set the date the opsim run was created. 

mafComment : str, opt 

Set a comment to describe the MAF analysis. 

mafVersion : str, opt 

Set the version of MAF used for analysis. 

mafDate : str, opt 

Set the date the MAF analysis was run. 

mafDir : str, opt 

The relative path to the MAF directory. 

dbFile : str, opt 

The relative path to the Opsim SQLite database file. 

mafRunId : int, opt 

The MafRunID to assign to this record in the database (note this is a primary key!). 

If this run (ie the mafDir) exists in the database already, this will be ignored.  

 

Returns 

------- 

int 

The mafRunID stored in the database. 

""" 

if opsimGroup is None: 

opsimGroup = 'NULL' 

if opsimRun is None: 

opsimRun = 'NULL' 

if opsimComment is None: 

opsimComment = 'NULL' 

if opsimVersion is None: 

opsimVersion = 'NULL' 

if opsimDate is None: 

opsimDate = 'NULL' 

if mafComment is None: 

mafComment = 'NULL' 

if mafVersion is None: 

mafVersion = 'NULL' 

if mafDate is None: 

mafDate = 'NULL' 

if mafDir is None: 

mafDir = 'NULL' 

if dbFile is None: 

dbFile = 'NULL' 

# Test if mafDir already exists in database. 

prevrun = self.session.query(RunRow).filter_by(mafDir=mafDir).all() 

if len(prevrun) > 0: 

runIds = [] 

for run in prevrun: 

runIds.append(run.mafRunId) 

print('Updating information in tracking database - %s already present with runId %s.' 

% (mafDir, runIds)) 

for run in prevrun: 

self.session.delete(run) 

self.session.commit() 

runinfo = RunRow(mafRunId=runIds[0], opsimGroup=opsimGroup, opsimRun=opsimRun, 

opsimComment=opsimComment, opsimVersion=opsimVersion, opsimDate=opsimDate, 

mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate, 

mafDir=mafDir, dbFile=dbFile) 

else: 

if mafRunId is not None: 

# Check if mafRunId exists already. 

existing = self.session.query(RunRow).filter_by(mafRunId=mafRunId).all() 

if len(existing) > 0: 

raise ValueError('MafRunId %d already exists in database, for %s. ' \ 

'Record must be deleted first.' 

% (mafRunId, existing[0].mafDir)) 

runinfo = RunRow(mafRunId=mafRunId, opsimGroup=opsimGroup, opsimRun=opsimRun, 

opsimComment=opsimComment, opsimVersion=opsimVersion, opsimDate=opsimDate, 

mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate, 

mafDir=mafDir, dbFile=dbFile) 

else: 

runinfo = RunRow(opsimGroup=opsimGroup, opsimRun=opsimRun, 

opsimComment=opsimComment, opsimVersion=opsimVersion, opsimDate=opsimDate, 

mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate, 

mafDir=mafDir, dbFile=dbFile) 

self.session.add(runinfo) 

self.session.commit() 

return runinfo.mafRunId 

 

def delRun(self, runId): 

""" 

Remove a run from the tracking database. 

""" 

runinfo = self.session.query(RunRow).filter_by(mafRunId=runId).all() 

if len(runinfo) == 0: 

raise Exception('Could not find run with mafRunId %d' %(runId)) 

if len(runinfo) > 1: 

raise Exception('Found more than one run with mafRunId %d' %(runId)) 

print('Removing run info for runId %d ' %(runId)) 

print(' ', runinfo) 

self.session.delete(runinfo[0]) 

self.session.commit() 

 

 

def addRunToDatabase(mafDir, trackingDbFile, opsimGroup=None, 

opsimRun=None, opsimComment=None, 

mafComment=None, dbFile=None): 

"""Adds information about a MAF analysis run to a MAF tracking database. 

 

Parameters 

---------- 

mafDir : str 

Path to the directory where the MAF results are located. 

trackingDb : str or lsst.sims.maf.TrackingDb 

Full filename (+path) to the tracking database storing the MAF run information or 

a TrackingDb object. 

opsimGroup: str, opt 

Name to use to group this run with other opsim runs. Default None. 

opsimRun : str, opt 

Name of the opsim run. If not provided, will attempt to use runName from confSummary.txt. 

opsimComment : str, opt 

Comment about the opsim run. If not provided, will attempt to use runComment from confSummary.txt. 

mafComment : str, opt 

Comment about the MAF analysis. If not provided, no comment will be recorded. 

dbFile : str, opt 

Relative path + name of the opsim database file. If not provided, no location will be recorded. 

""" 

mafDir = os.path.abspath(mafDir) 

if not os.path.isdir(mafDir): 

raise ValueError('There is no directory containing MAF outputs at %s.' % (mafDir)) 

 

trackingDb = TrackingDb(database=trackingDbFile) 

autoOpsimRun = None 

autoOpsimComment = None 

opsimVersion = None 

opsimDate = None 

mafVersion = None 

mafDate = None 

if os.path.isfile(os.path.join(mafDir, 'configSummary.txt')): 

file = open(os.path.join(mafDir, 'configSummary.txt')) 

for line in file: 

tmp = line.split() 

if tmp[0].startswith('RunName'): 

autoOpsimRun = ' '.join(tmp[1:]) 

if tmp[0].startswith('RunComment'): 

autoOpsimComment = ' '.join(tmp[1:]) 

# MAF Date may be in a line with "MafDate" (new configs) 

# or at the end of "MAFVersion" (old configs). 

if tmp[0].startswith('MAFDate'): 

mafDate = tmp[-1] 

if tmp[0].startswith('MAFVersion'): 

mafVersion = tmp[1] 

if len(tmp) > 2: 

mafDate = tmp[-1] 

if tmp[0].startswith('OpsimDate'): 

opsimDate = tmp[-1] 

if len(tmp) > 2: 

opsimDate = tmp[-2] 

if tmp[0].startswith('OpsimVersion'): 

opsimVersion = tmp[1] 

if len(tmp) > 2: 

opsimDate = tmp[-2] 

# And convert formats to '-' (again, multiple versions of configs). 

if mafDate is not None: 

if len(mafDate.split('/')) > 1: 

t = mafDate.split('/') 

if len(t[2]) == 2: 

t[2] = '20' + t[2] 

mafDate = '-'.join([t[2], t[1], t[0]]) 

if opsimDate is not None: 

if len(opsimDate.split('/')) > 1: 

t = opsimDate.split('/') 

if len(t[2]) == 2: 

t[2] = '20' + t[2] 

opsimDate = '-'.join([t[2], t[1], t[0]]) 

 

if opsimRun is None: 

opsimRun = autoOpsimRun 

if opsimComment is None: 

opsimComment = autoOpsimComment 

 

print('Adding to tracking database at %s:' % (trackingDbFile)) 

print(' MafDir = %s' % (mafDir)) 

print(' MafComment = %s' % (mafComment)) 

print(' OpsimGroup = %s' % (opsimGroup)) 

print(' OpsimRun = %s' % (opsimRun)) 

print(' OpsimComment = %s' % (opsimComment)) 

print(' OpsimVersion = %s' % (opsimVersion)) 

print(' OpsimDate = %s' % (opsimDate)) 

print(' MafVersion = %s' % (mafVersion)) 

print(' MafDate = %s' % (mafDate)) 

print(' Opsim dbFile = %s' % (dbFile)) 

runId = trackingDb.addRun(opsimGroup=opsimGroup, opsimRun=opsimRun, opsimComment=opsimComment, 

opsimVersion=opsimVersion, opsimDate=opsimDate, 

mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate, 

mafDir=mafDir, dbFile=dbFile) 

print('Used MAF RunID %d' % (runId)) 

trackingDb.close()