Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from __future__ import print_function 

2from builtins import str 

3from builtins import object 

4import os 

5from sqlalchemy import create_engine, Column, Integer, String 

6from sqlalchemy.engine import url 

7from sqlalchemy.orm import sessionmaker 

8 

9from sqlalchemy.ext.declarative import declarative_base 

10from sqlalchemy.exc import DatabaseError 

11from lsst.daf.persistence import DbAuth 

12 

13Base = declarative_base() 

14 

15__all__ = ['TrackingDb', 'addRunToDatabase'] 

16 

17 

18class RunRow(Base): 

19 """ 

20 Define contents and format of run list table. 

21 

22 Table to list all available MAF results, along with their opsim run and some comment info. 

23 """ 

24 __tablename__ = "runs" 

25 # Define columns in metric list table. 

26 mafRunId = Column(Integer, primary_key=True) 

27 opsimGroup = Column(String) 

28 opsimRun = Column(String) 

29 opsimComment = Column(String) 

30 opsimVersion = Column(String) 

31 opsimDate = Column(String) 

32 dbFile = Column(String) 

33 mafComment = Column(String) 

34 mafVersion = Column(String) 

35 mafDate = Column(String) 

36 mafDir = Column(String) 

37 def __repr__(self): 

38 rstr = "<Run(mafRunId='%d', opsimGroup='%s', opsimRun='%s', opsimComment='%s', " \ 

39 "opsimVersion='%s', opsimDate='%s', mafComment='%s', " \ 

40 "mafVersion='%s', mafDate='%s', mafDir='%s', dbFile='%s'>" \ 

41 % (self.mafRunId, self.opsimGroup, self.opsimRun, self.opsimComment, 

42 self.opsimVersion, self.opsimDate, self.mafComment, self.mafVersion, self.mafDate, 

43 self.mafDir, self.dbFile) 

44 return rstr 

45 

46 

47class TrackingDb(object): 

48 

49 def __init__(self, database=None, driver='sqlite', host=None, port=None, 

50 trackingDbverbose=False): 

51 """ 

52 Instantiate the results database, creating metrics, plots and summarystats tables. 

53 """ 

54 self.verbose = trackingDbverbose 

55 # Connect to database 

56 # for sqlite, connecting to non-existent database creates it automatically 

57 if database is None: 

58 # Default is a file in the current directory. 

59 self.database = os.path.join(os.getcwd(), 'trackingDb_sqlite.db') 

60 self.driver = 'sqlite' 

61 else: 

62 self.database = database 

63 self.driver = driver 

64 self.host = host 

65 self.port = port 

66 

67 if self.driver == 'sqlite': 

68 dbAddress = url.URL(drivername=self.driver, database=self.database) 

69 else: 

70 dbAddress = url.URL(self.driver, 

71 username=DbAuth.username(self.host, str(self.port)), 

72 password=DbAuth.password(self.host, str(self.port)), 

73 host=self.host, 

74 port=self.port, 

75 database=self.database) 

76 

77 engine = create_engine(dbAddress, echo=self.verbose) 

78 if self.verbose: 

79 print('Created or connected to MAF tracking %s database at %s' %(self.driver, self.database)) 

80 self.Session = sessionmaker(bind=engine) 

81 self.session = self.Session() 

82 # Create the tables, if they don't already exist. 

83 try: 

84 Base.metadata.create_all(engine) 

85 except DatabaseError: 

86 raise DatabaseError("Cannot create a %s database at %s. Check directory exists." %(self.driver, self.database)) 

87 

88 def close(self): 

89 self.session.close() 

90 

91 def addRun(self, opsimGroup=None, opsimRun=None, opsimComment=None, opsimVersion=None, opsimDate=None, 

92 mafComment=None, mafVersion=None, mafDate=None, mafDir=None, dbFile=None, mafRunId=None): 

93 """Add a run to the tracking database. 

94  

95 Parameters 

96 ---------- 

97 opsimGroup : str, opt 

98 Set a name to group this run with (eg. "Tier 1, 2016"). 

99 opsimRun : str, opt 

100 Set a name for the opsim run. 

101 opsimComment : str, opt 

102 Set a comment describing the opsim run. 

103 opsimVersion : str, opt 

104 Set the version of opsim. 

105 opsimDate : str, opt 

106 Set the date the opsim run was created. 

107 mafComment : str, opt 

108 Set a comment to describe the MAF analysis. 

109 mafVersion : str, opt 

110 Set the version of MAF used for analysis. 

111 mafDate : str, opt 

112 Set the date the MAF analysis was run. 

113 mafDir : str, opt 

114 The relative path to the MAF directory. 

115 dbFile : str, opt 

116 The relative path to the Opsim SQLite database file. 

117 mafRunId : int, opt 

118 The MafRunID to assign to this record in the database (note this is a primary key!). 

119 If this run (ie the mafDir) exists in the database already, this will be ignored.  

120  

121 Returns 

122 ------- 

123 int 

124 The mafRunID stored in the database. 

125 """ 

126 if opsimGroup is None: 

127 opsimGroup = 'NULL' 

128 if opsimRun is None: 

129 opsimRun = 'NULL' 

130 if opsimComment is None: 

131 opsimComment = 'NULL' 

132 if opsimVersion is None: 

133 opsimVersion = 'NULL' 

134 if opsimDate is None: 

135 opsimDate = 'NULL' 

136 if mafComment is None: 

137 mafComment = 'NULL' 

138 if mafVersion is None: 

139 mafVersion = 'NULL' 

140 if mafDate is None: 

141 mafDate = 'NULL' 

142 if mafDir is None: 

143 mafDir = 'NULL' 

144 if dbFile is None: 

145 dbFile = 'NULL' 

146 # Test if mafDir already exists in database. 

147 prevrun = self.session.query(RunRow).filter_by(mafDir=mafDir).all() 

148 if len(prevrun) > 0: 

149 runIds = [] 

150 for run in prevrun: 

151 runIds.append(run.mafRunId) 

152 print('Updating information in tracking database - %s already present with runId %s.' 

153 % (mafDir, runIds)) 

154 for run in prevrun: 

155 self.session.delete(run) 

156 self.session.commit() 

157 runinfo = RunRow(mafRunId=runIds[0], opsimGroup=opsimGroup, opsimRun=opsimRun, 

158 opsimComment=opsimComment, opsimVersion=opsimVersion, opsimDate=opsimDate, 

159 mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate, 

160 mafDir=mafDir, dbFile=dbFile) 

161 else: 

162 if mafRunId is not None: 

163 # Check if mafRunId exists already. 

164 existing = self.session.query(RunRow).filter_by(mafRunId=mafRunId).all() 

165 if len(existing) > 0: 

166 raise ValueError('MafRunId %d already exists in database, for %s. ' \ 

167 'Record must be deleted first.' 

168 % (mafRunId, existing[0].mafDir)) 

169 runinfo = RunRow(mafRunId=mafRunId, opsimGroup=opsimGroup, opsimRun=opsimRun, 

170 opsimComment=opsimComment, opsimVersion=opsimVersion, opsimDate=opsimDate, 

171 mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate, 

172 mafDir=mafDir, dbFile=dbFile) 

173 else: 

174 runinfo = RunRow(opsimGroup=opsimGroup, opsimRun=opsimRun, 

175 opsimComment=opsimComment, opsimVersion=opsimVersion, opsimDate=opsimDate, 

176 mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate, 

177 mafDir=mafDir, dbFile=dbFile) 

178 self.session.add(runinfo) 

179 self.session.commit() 

180 return runinfo.mafRunId 

181 

182 def delRun(self, runId): 

183 """ 

184 Remove a run from the tracking database. 

185 """ 

186 runinfo = self.session.query(RunRow).filter_by(mafRunId=runId).all() 

187 if len(runinfo) == 0: 

188 raise Exception('Could not find run with mafRunId %d' %(runId)) 

189 if len(runinfo) > 1: 

190 raise Exception('Found more than one run with mafRunId %d' %(runId)) 

191 print('Removing run info for runId %d ' %(runId)) 

192 print(' ', runinfo) 

193 self.session.delete(runinfo[0]) 

194 self.session.commit() 

195 

196 

197def addRunToDatabase(mafDir, trackingDbFile, opsimGroup=None, 

198 opsimRun=None, opsimComment=None, 

199 mafComment=None, dbFile=None): 

200 """Adds information about a MAF analysis run to a MAF tracking database. 

201 

202 Parameters 

203 ---------- 

204 mafDir : str 

205 Path to the directory where the MAF results are located. 

206 trackingDb : str or lsst.sims.maf.TrackingDb 

207 Full filename (+path) to the tracking database storing the MAF run information or 

208 a TrackingDb object. 

209 opsimGroup: str, opt 

210 Name to use to group this run with other opsim runs. Default None. 

211 opsimRun : str, opt 

212 Name of the opsim run. If not provided, will attempt to use runName from confSummary.txt. 

213 opsimComment : str, opt 

214 Comment about the opsim run. If not provided, will attempt to use runComment from confSummary.txt. 

215 mafComment : str, opt 

216 Comment about the MAF analysis. If not provided, no comment will be recorded. 

217 dbFile : str, opt 

218 Relative path + name of the opsim database file. If not provided, no location will be recorded. 

219 """ 

220 mafDir = os.path.abspath(mafDir) 

221 if not os.path.isdir(mafDir): 

222 raise ValueError('There is no directory containing MAF outputs at %s.' % (mafDir)) 

223 

224 trackingDb = TrackingDb(database=trackingDbFile) 

225 autoOpsimRun = None 

226 autoOpsimComment = None 

227 opsimVersion = None 

228 opsimDate = None 

229 mafVersion = None 

230 mafDate = None 

231 if os.path.isfile(os.path.join(mafDir, 'configSummary.txt')): 

232 file = open(os.path.join(mafDir, 'configSummary.txt')) 

233 for line in file: 

234 tmp = line.split() 

235 if tmp[0].startswith('RunName'): 

236 autoOpsimRun = ' '.join(tmp[1:]) 

237 if tmp[0].startswith('RunComment'): 

238 autoOpsimComment = ' '.join(tmp[1:]) 

239 # MAF Date may be in a line with "MafDate" (new configs) 

240 # or at the end of "MAFVersion" (old configs). 

241 if tmp[0].startswith('MAFDate'): 

242 mafDate = tmp[-1] 

243 if tmp[0].startswith('MAFVersion'): 

244 mafVersion = tmp[1] 

245 if len(tmp) > 2: 

246 mafDate = tmp[-1] 

247 if tmp[0].startswith('OpsimDate'): 

248 opsimDate = tmp[-1] 

249 if len(tmp) > 2: 

250 opsimDate = tmp[-2] 

251 if tmp[0].startswith('OpsimVersion'): 

252 opsimVersion = tmp[1] 

253 if len(tmp) > 2: 

254 opsimDate = tmp[-2] 

255 # And convert formats to '-' (again, multiple versions of configs). 

256 if mafDate is not None: 

257 if len(mafDate.split('/')) > 1: 

258 t = mafDate.split('/') 

259 if len(t[2]) == 2: 

260 t[2] = '20' + t[2] 

261 mafDate = '-'.join([t[2], t[1], t[0]]) 

262 if opsimDate is not None: 

263 if len(opsimDate.split('/')) > 1: 

264 t = opsimDate.split('/') 

265 if len(t[2]) == 2: 

266 t[2] = '20' + t[2] 

267 opsimDate = '-'.join([t[2], t[1], t[0]]) 

268 

269 if opsimRun is None: 

270 opsimRun = autoOpsimRun 

271 if opsimComment is None: 

272 opsimComment = autoOpsimComment 

273 

274 print('Adding to tracking database at %s:' % (trackingDbFile)) 

275 print(' MafDir = %s' % (mafDir)) 

276 print(' MafComment = %s' % (mafComment)) 

277 print(' OpsimGroup = %s' % (opsimGroup)) 

278 print(' OpsimRun = %s' % (opsimRun)) 

279 print(' OpsimComment = %s' % (opsimComment)) 

280 print(' OpsimVersion = %s' % (opsimVersion)) 

281 print(' OpsimDate = %s' % (opsimDate)) 

282 print(' MafVersion = %s' % (mafVersion)) 

283 print(' MafDate = %s' % (mafDate)) 

284 print(' Opsim dbFile = %s' % (dbFile)) 

285 runId = trackingDb.addRun(opsimGroup=opsimGroup, opsimRun=opsimRun, opsimComment=opsimComment, 

286 opsimVersion=opsimVersion, opsimDate=opsimDate, 

287 mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate, 

288 mafDir=mafDir, dbFile=dbFile) 

289 print('Used MAF RunID %d' % (runId)) 

290 trackingDb.close()