Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from __future__ import print_function 

2from builtins import str 

3from builtins import object 

4import os 

5from sqlalchemy import create_engine, Column, Integer, String 

6from sqlalchemy.engine import url 

7from sqlalchemy.orm import sessionmaker 

8 

9from sqlalchemy.ext.declarative import declarative_base 

10from sqlalchemy.exc import DatabaseError 

11 

12Base = declarative_base() 

13 

14__all__ = ['TrackingDb', 'addRunToDatabase'] 

15 

16 

17class RunRow(Base): 

18 """ 

19 Define contents and format of run list table. 

20 

21 Table to list all available MAF results, along with their opsim run and some comment info. 

22 """ 

23 __tablename__ = "runs" 

24 # Define columns in metric list table. 

25 mafRunId = Column(Integer, primary_key=True) 

26 opsimGroup = Column(String) 

27 opsimRun = Column(String) 

28 opsimComment = Column(String) 

29 opsimVersion = Column(String) 

30 opsimDate = Column(String) 

31 dbFile = Column(String) 

32 mafComment = Column(String) 

33 mafVersion = Column(String) 

34 mafDate = Column(String) 

35 mafDir = Column(String) 

36 def __repr__(self): 

37 rstr = "<Run(mafRunId='%d', opsimGroup='%s', opsimRun='%s', opsimComment='%s', " \ 

38 "opsimVersion='%s', opsimDate='%s', mafComment='%s', " \ 

39 "mafVersion='%s', mafDate='%s', mafDir='%s', dbFile='%s'>" \ 

40 % (self.mafRunId, self.opsimGroup, self.opsimRun, self.opsimComment, 

41 self.opsimVersion, self.opsimDate, self.mafComment, self.mafVersion, self.mafDate, 

42 self.mafDir, self.dbFile) 

43 return rstr 

44 

45 

46class TrackingDb(object): 

47 """Sqlite database to track MAF output runs and their locations, for showMaf.py 

48 """ 

49 def __init__(self, database=None, trackingDbverbose=False): 

50 """ 

51 Instantiate the results database, creating metrics, plots and summarystats tables. 

52 """ 

53 self.verbose = trackingDbverbose 

54 self.driver = 'sqlite' 

55 # Connect to database 

56 # for sqlite, connecting to non-existent database creates it automatically 

57 if database is None: 

58 # Default is a file in the current directory. 

59 self.database = os.path.join(os.getcwd(), 'trackingDb_sqlite.db') 

60 else: 

61 self.database = database 

62 # only sqlite 

63 dbAddress = url.URL(drivername=self.driver, database=self.database) 

64 engine = create_engine(dbAddress, echo=self.verbose) 

65 if self.verbose: 

66 print('Created or connected to MAF tracking %s database at %s' %(self.driver, self.database)) 

67 self.Session = sessionmaker(bind=engine) 

68 self.session = self.Session() 

69 # Create the tables, if they don't already exist. 

70 try: 

71 Base.metadata.create_all(engine) 

72 except DatabaseError: 

73 raise DatabaseError("Cannot create a %s database at %s. Check directory exists." %(self.driver, 

74 self.database)) 

75 

76 def close(self): 

77 self.session.close() 

78 

79 def addRun(self, opsimGroup=None, opsimRun=None, opsimComment=None, opsimVersion=None, opsimDate=None, 

80 mafComment=None, mafVersion=None, mafDate=None, mafDir=None, dbFile=None, mafRunId=None): 

81 """Add a run to the tracking database. 

82  

83 Parameters 

84 ---------- 

85 opsimGroup : str, opt 

86 Set a name to group this run with (eg. "Tier 1, 2016"). 

87 opsimRun : str, opt 

88 Set a name for the opsim run. 

89 opsimComment : str, opt 

90 Set a comment describing the opsim run. 

91 opsimVersion : str, opt 

92 Set the version of opsim. 

93 opsimDate : str, opt 

94 Set the date the opsim run was created. 

95 mafComment : str, opt 

96 Set a comment to describe the MAF analysis. 

97 mafVersion : str, opt 

98 Set the version of MAF used for analysis. 

99 mafDate : str, opt 

100 Set the date the MAF analysis was run. 

101 mafDir : str, opt 

102 The relative path to the MAF directory. 

103 dbFile : str, opt 

104 The relative path to the Opsim SQLite database file. 

105 mafRunId : int, opt 

106 The MafRunID to assign to this record in the database (note this is a primary key!). 

107 If this run (ie the mafDir) exists in the database already, this will be ignored.  

108  

109 Returns 

110 ------- 

111 int 

112 The mafRunID stored in the database. 

113 """ 

114 if opsimGroup is None: 

115 opsimGroup = 'NULL' 

116 if opsimRun is None: 

117 opsimRun = 'NULL' 

118 if opsimComment is None: 

119 opsimComment = 'NULL' 

120 if opsimVersion is None: 

121 opsimVersion = 'NULL' 

122 if opsimDate is None: 

123 opsimDate = 'NULL' 

124 if mafComment is None: 

125 mafComment = 'NULL' 

126 if mafVersion is None: 

127 mafVersion = 'NULL' 

128 if mafDate is None: 

129 mafDate = 'NULL' 

130 if mafDir is None: 

131 mafDir = 'NULL' 

132 if dbFile is None: 

133 dbFile = 'NULL' 

134 # Test if mafDir already exists in database. 

135 prevrun = self.session.query(RunRow).filter_by(mafDir=mafDir).all() 

136 if len(prevrun) > 0: 

137 runIds = [] 

138 for run in prevrun: 

139 runIds.append(run.mafRunId) 

140 print('Updating information in tracking database - %s already present with runId %s.' 

141 % (mafDir, runIds)) 

142 for run in prevrun: 

143 self.session.delete(run) 

144 self.session.commit() 

145 runinfo = RunRow(mafRunId=runIds[0], opsimGroup=opsimGroup, opsimRun=opsimRun, 

146 opsimComment=opsimComment, opsimVersion=opsimVersion, opsimDate=opsimDate, 

147 mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate, 

148 mafDir=mafDir, dbFile=dbFile) 

149 else: 

150 if mafRunId is not None: 

151 # Check if mafRunId exists already. 

152 existing = self.session.query(RunRow).filter_by(mafRunId=mafRunId).all() 

153 if len(existing) > 0: 

154 raise ValueError('MafRunId %d already exists in database, for %s. ' \ 

155 'Record must be deleted first.' 

156 % (mafRunId, existing[0].mafDir)) 

157 runinfo = RunRow(mafRunId=mafRunId, opsimGroup=opsimGroup, opsimRun=opsimRun, 

158 opsimComment=opsimComment, opsimVersion=opsimVersion, opsimDate=opsimDate, 

159 mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate, 

160 mafDir=mafDir, dbFile=dbFile) 

161 else: 

162 runinfo = RunRow(opsimGroup=opsimGroup, opsimRun=opsimRun, 

163 opsimComment=opsimComment, opsimVersion=opsimVersion, opsimDate=opsimDate, 

164 mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate, 

165 mafDir=mafDir, dbFile=dbFile) 

166 self.session.add(runinfo) 

167 self.session.commit() 

168 return runinfo.mafRunId 

169 

170 def delRun(self, runId): 

171 """ 

172 Remove a run from the tracking database. 

173 """ 

174 runinfo = self.session.query(RunRow).filter_by(mafRunId=runId).all() 

175 if len(runinfo) == 0: 

176 raise Exception('Could not find run with mafRunId %d' %(runId)) 

177 if len(runinfo) > 1: 

178 raise Exception('Found more than one run with mafRunId %d' %(runId)) 

179 print('Removing run info for runId %d ' %(runId)) 

180 print(' ', runinfo) 

181 self.session.delete(runinfo[0]) 

182 self.session.commit() 

183 

184 

185def addRunToDatabase(mafDir, trackingDbFile, opsimGroup=None, 

186 opsimRun=None, opsimComment=None, 

187 mafComment=None, dbFile=None): 

188 """Adds information about a MAF analysis run to a MAF tracking database. 

189 

190 Parameters 

191 ---------- 

192 mafDir : str 

193 Path to the directory where the MAF results are located. 

194 trackingDb : str or lsst.sims.maf.TrackingDb 

195 Full filename (+path) to the tracking database storing the MAF run information or 

196 a TrackingDb object. 

197 opsimGroup: str, opt 

198 Name to use to group this run with other opsim runs. Default None. 

199 opsimRun : str, opt 

200 Name of the opsim run. If not provided, will attempt to use runName from confSummary.txt. 

201 opsimComment : str, opt 

202 Comment about the opsim run. If not provided, will attempt to use runComment from confSummary.txt. 

203 mafComment : str, opt 

204 Comment about the MAF analysis. If not provided, no comment will be recorded. 

205 dbFile : str, opt 

206 Relative path + name of the opsim database file. If not provided, no location will be recorded. 

207 """ 

208 mafDir = os.path.abspath(mafDir) 

209 if not os.path.isdir(mafDir): 

210 raise ValueError('There is no directory containing MAF outputs at %s.' % (mafDir)) 

211 

212 trackingDb = TrackingDb(database=trackingDbFile) 

213 autoOpsimRun = None 

214 autoOpsimComment = None 

215 opsimVersion = None 

216 opsimDate = None 

217 mafVersion = None 

218 mafDate = None 

219 if os.path.isfile(os.path.join(mafDir, 'configSummary.txt')): 

220 file = open(os.path.join(mafDir, 'configSummary.txt')) 

221 for line in file: 

222 tmp = line.split() 

223 if tmp[0].startswith('RunName'): 

224 autoOpsimRun = ' '.join(tmp[1:]) 

225 if tmp[0].startswith('RunComment'): 

226 autoOpsimComment = ' '.join(tmp[1:]) 

227 # MAF Date may be in a line with "MafDate" (new configs) 

228 # or at the end of "MAFVersion" (old configs). 

229 if tmp[0].startswith('MAFDate'): 

230 mafDate = tmp[-1] 

231 if tmp[0].startswith('MAFVersion'): 

232 mafVersion = tmp[1] 

233 if len(tmp) > 2: 

234 mafDate = tmp[-1] 

235 if tmp[0].startswith('OpsimDate'): 

236 opsimDate = tmp[-1] 

237 if len(tmp) > 2: 

238 opsimDate = tmp[-2] 

239 if tmp[0].startswith('OpsimVersion'): 

240 opsimVersion = tmp[1] 

241 if len(tmp) > 2: 

242 opsimDate = tmp[-2] 

243 # And convert formats to '-' (again, multiple versions of configs). 

244 if mafDate is not None: 

245 if len(mafDate.split('/')) > 1: 

246 t = mafDate.split('/') 

247 if len(t[2]) == 2: 

248 t[2] = '20' + t[2] 

249 mafDate = '-'.join([t[2], t[1], t[0]]) 

250 if opsimDate is not None: 

251 if len(opsimDate.split('/')) > 1: 

252 t = opsimDate.split('/') 

253 if len(t[2]) == 2: 

254 t[2] = '20' + t[2] 

255 opsimDate = '-'.join([t[2], t[1], t[0]]) 

256 

257 if opsimRun is None: 

258 opsimRun = autoOpsimRun 

259 if opsimComment is None: 

260 opsimComment = autoOpsimComment 

261 

262 print('Adding to tracking database at %s:' % (trackingDbFile)) 

263 print(' MafDir = %s' % (mafDir)) 

264 print(' MafComment = %s' % (mafComment)) 

265 print(' OpsimGroup = %s' % (opsimGroup)) 

266 print(' OpsimRun = %s' % (opsimRun)) 

267 print(' OpsimComment = %s' % (opsimComment)) 

268 print(' OpsimVersion = %s' % (opsimVersion)) 

269 print(' OpsimDate = %s' % (opsimDate)) 

270 print(' MafVersion = %s' % (mafVersion)) 

271 print(' MafDate = %s' % (mafDate)) 

272 print(' Opsim dbFile = %s' % (dbFile)) 

273 runId = trackingDb.addRun(opsimGroup=opsimGroup, opsimRun=opsimRun, opsimComment=opsimComment, 

274 opsimVersion=opsimVersion, opsimDate=opsimDate, 

275 mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate, 

276 mafDir=mafDir, dbFile=dbFile) 

277 print('Used MAF RunID %d' % (runId)) 

278 trackingDb.close()