Coverage for python/lsst/sims/maf/db/trackingDb.py : 13%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import print_function
2from builtins import str
3from builtins import object
4import os
5from sqlalchemy import create_engine, Column, Integer, String
6from sqlalchemy.engine import url
7from sqlalchemy.orm import sessionmaker
9from sqlalchemy.ext.declarative import declarative_base
10from sqlalchemy.exc import DatabaseError
11from lsst.daf.persistence import DbAuth
13Base = declarative_base()
15__all__ = ['TrackingDb', 'addRunToDatabase']
18class RunRow(Base):
19 """
20 Define contents and format of run list table.
22 Table to list all available MAF results, along with their opsim run and some comment info.
23 """
24 __tablename__ = "runs"
25 # Define columns in metric list table.
26 mafRunId = Column(Integer, primary_key=True)
27 opsimGroup = Column(String)
28 opsimRun = Column(String)
29 opsimComment = Column(String)
30 opsimVersion = Column(String)
31 opsimDate = Column(String)
32 dbFile = Column(String)
33 mafComment = Column(String)
34 mafVersion = Column(String)
35 mafDate = Column(String)
36 mafDir = Column(String)
37 def __repr__(self):
38 rstr = "<Run(mafRunId='%d', opsimGroup='%s', opsimRun='%s', opsimComment='%s', " \
39 "opsimVersion='%s', opsimDate='%s', mafComment='%s', " \
40 "mafVersion='%s', mafDate='%s', mafDir='%s', dbFile='%s'>" \
41 % (self.mafRunId, self.opsimGroup, self.opsimRun, self.opsimComment,
42 self.opsimVersion, self.opsimDate, self.mafComment, self.mafVersion, self.mafDate,
43 self.mafDir, self.dbFile)
44 return rstr
47class TrackingDb(object):
49 def __init__(self, database=None, driver='sqlite', host=None, port=None,
50 trackingDbverbose=False):
51 """
52 Instantiate the results database, creating metrics, plots and summarystats tables.
53 """
54 self.verbose = trackingDbverbose
55 # Connect to database
56 # for sqlite, connecting to non-existent database creates it automatically
57 if database is None:
58 # Default is a file in the current directory.
59 self.database = os.path.join(os.getcwd(), 'trackingDb_sqlite.db')
60 self.driver = 'sqlite'
61 else:
62 self.database = database
63 self.driver = driver
64 self.host = host
65 self.port = port
67 if self.driver == 'sqlite':
68 dbAddress = url.URL(drivername=self.driver, database=self.database)
69 else:
70 dbAddress = url.URL(self.driver,
71 username=DbAuth.username(self.host, str(self.port)),
72 password=DbAuth.password(self.host, str(self.port)),
73 host=self.host,
74 port=self.port,
75 database=self.database)
77 engine = create_engine(dbAddress, echo=self.verbose)
78 if self.verbose:
79 print('Created or connected to MAF tracking %s database at %s' %(self.driver, self.database))
80 self.Session = sessionmaker(bind=engine)
81 self.session = self.Session()
82 # Create the tables, if they don't already exist.
83 try:
84 Base.metadata.create_all(engine)
85 except DatabaseError:
86 raise DatabaseError("Cannot create a %s database at %s. Check directory exists." %(self.driver, self.database))
88 def close(self):
89 self.session.close()
91 def addRun(self, opsimGroup=None, opsimRun=None, opsimComment=None, opsimVersion=None, opsimDate=None,
92 mafComment=None, mafVersion=None, mafDate=None, mafDir=None, dbFile=None, mafRunId=None):
93 """Add a run to the tracking database.
95 Parameters
96 ----------
97 opsimGroup : str, opt
98 Set a name to group this run with (eg. "Tier 1, 2016").
99 opsimRun : str, opt
100 Set a name for the opsim run.
101 opsimComment : str, opt
102 Set a comment describing the opsim run.
103 opsimVersion : str, opt
104 Set the version of opsim.
105 opsimDate : str, opt
106 Set the date the opsim run was created.
107 mafComment : str, opt
108 Set a comment to describe the MAF analysis.
109 mafVersion : str, opt
110 Set the version of MAF used for analysis.
111 mafDate : str, opt
112 Set the date the MAF analysis was run.
113 mafDir : str, opt
114 The relative path to the MAF directory.
115 dbFile : str, opt
116 The relative path to the Opsim SQLite database file.
117 mafRunId : int, opt
118 The MafRunID to assign to this record in the database (note this is a primary key!).
119 If this run (ie the mafDir) exists in the database already, this will be ignored.
121 Returns
122 -------
123 int
124 The mafRunID stored in the database.
125 """
126 if opsimGroup is None:
127 opsimGroup = 'NULL'
128 if opsimRun is None:
129 opsimRun = 'NULL'
130 if opsimComment is None:
131 opsimComment = 'NULL'
132 if opsimVersion is None:
133 opsimVersion = 'NULL'
134 if opsimDate is None:
135 opsimDate = 'NULL'
136 if mafComment is None:
137 mafComment = 'NULL'
138 if mafVersion is None:
139 mafVersion = 'NULL'
140 if mafDate is None:
141 mafDate = 'NULL'
142 if mafDir is None:
143 mafDir = 'NULL'
144 if dbFile is None:
145 dbFile = 'NULL'
146 # Test if mafDir already exists in database.
147 prevrun = self.session.query(RunRow).filter_by(mafDir=mafDir).all()
148 if len(prevrun) > 0:
149 runIds = []
150 for run in prevrun:
151 runIds.append(run.mafRunId)
152 print('Updating information in tracking database - %s already present with runId %s.'
153 % (mafDir, runIds))
154 for run in prevrun:
155 self.session.delete(run)
156 self.session.commit()
157 runinfo = RunRow(mafRunId=runIds[0], opsimGroup=opsimGroup, opsimRun=opsimRun,
158 opsimComment=opsimComment, opsimVersion=opsimVersion, opsimDate=opsimDate,
159 mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate,
160 mafDir=mafDir, dbFile=dbFile)
161 else:
162 if mafRunId is not None:
163 # Check if mafRunId exists already.
164 existing = self.session.query(RunRow).filter_by(mafRunId=mafRunId).all()
165 if len(existing) > 0:
166 raise ValueError('MafRunId %d already exists in database, for %s. ' \
167 'Record must be deleted first.'
168 % (mafRunId, existing[0].mafDir))
169 runinfo = RunRow(mafRunId=mafRunId, opsimGroup=opsimGroup, opsimRun=opsimRun,
170 opsimComment=opsimComment, opsimVersion=opsimVersion, opsimDate=opsimDate,
171 mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate,
172 mafDir=mafDir, dbFile=dbFile)
173 else:
174 runinfo = RunRow(opsimGroup=opsimGroup, opsimRun=opsimRun,
175 opsimComment=opsimComment, opsimVersion=opsimVersion, opsimDate=opsimDate,
176 mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate,
177 mafDir=mafDir, dbFile=dbFile)
178 self.session.add(runinfo)
179 self.session.commit()
180 return runinfo.mafRunId
182 def delRun(self, runId):
183 """
184 Remove a run from the tracking database.
185 """
186 runinfo = self.session.query(RunRow).filter_by(mafRunId=runId).all()
187 if len(runinfo) == 0:
188 raise Exception('Could not find run with mafRunId %d' %(runId))
189 if len(runinfo) > 1:
190 raise Exception('Found more than one run with mafRunId %d' %(runId))
191 print('Removing run info for runId %d ' %(runId))
192 print(' ', runinfo)
193 self.session.delete(runinfo[0])
194 self.session.commit()
197def addRunToDatabase(mafDir, trackingDbFile, opsimGroup=None,
198 opsimRun=None, opsimComment=None,
199 mafComment=None, dbFile=None):
200 """Adds information about a MAF analysis run to a MAF tracking database.
202 Parameters
203 ----------
204 mafDir : str
205 Path to the directory where the MAF results are located.
206 trackingDb : str or lsst.sims.maf.TrackingDb
207 Full filename (+path) to the tracking database storing the MAF run information or
208 a TrackingDb object.
209 opsimGroup: str, opt
210 Name to use to group this run with other opsim runs. Default None.
211 opsimRun : str, opt
212 Name of the opsim run. If not provided, will attempt to use runName from confSummary.txt.
213 opsimComment : str, opt
214 Comment about the opsim run. If not provided, will attempt to use runComment from confSummary.txt.
215 mafComment : str, opt
216 Comment about the MAF analysis. If not provided, no comment will be recorded.
217 dbFile : str, opt
218 Relative path + name of the opsim database file. If not provided, no location will be recorded.
219 """
220 mafDir = os.path.abspath(mafDir)
221 if not os.path.isdir(mafDir):
222 raise ValueError('There is no directory containing MAF outputs at %s.' % (mafDir))
224 trackingDb = TrackingDb(database=trackingDbFile)
225 autoOpsimRun = None
226 autoOpsimComment = None
227 opsimVersion = None
228 opsimDate = None
229 mafVersion = None
230 mafDate = None
231 if os.path.isfile(os.path.join(mafDir, 'configSummary.txt')):
232 file = open(os.path.join(mafDir, 'configSummary.txt'))
233 for line in file:
234 tmp = line.split()
235 if tmp[0].startswith('RunName'):
236 autoOpsimRun = ' '.join(tmp[1:])
237 if tmp[0].startswith('RunComment'):
238 autoOpsimComment = ' '.join(tmp[1:])
239 # MAF Date may be in a line with "MafDate" (new configs)
240 # or at the end of "MAFVersion" (old configs).
241 if tmp[0].startswith('MAFDate'):
242 mafDate = tmp[-1]
243 if tmp[0].startswith('MAFVersion'):
244 mafVersion = tmp[1]
245 if len(tmp) > 2:
246 mafDate = tmp[-1]
247 if tmp[0].startswith('OpsimDate'):
248 opsimDate = tmp[-1]
249 if len(tmp) > 2:
250 opsimDate = tmp[-2]
251 if tmp[0].startswith('OpsimVersion'):
252 opsimVersion = tmp[1]
253 if len(tmp) > 2:
254 opsimDate = tmp[-2]
255 # And convert formats to '-' (again, multiple versions of configs).
256 if mafDate is not None:
257 if len(mafDate.split('/')) > 1:
258 t = mafDate.split('/')
259 if len(t[2]) == 2:
260 t[2] = '20' + t[2]
261 mafDate = '-'.join([t[2], t[1], t[0]])
262 if opsimDate is not None:
263 if len(opsimDate.split('/')) > 1:
264 t = opsimDate.split('/')
265 if len(t[2]) == 2:
266 t[2] = '20' + t[2]
267 opsimDate = '-'.join([t[2], t[1], t[0]])
269 if opsimRun is None:
270 opsimRun = autoOpsimRun
271 if opsimComment is None:
272 opsimComment = autoOpsimComment
274 print('Adding to tracking database at %s:' % (trackingDbFile))
275 print(' MafDir = %s' % (mafDir))
276 print(' MafComment = %s' % (mafComment))
277 print(' OpsimGroup = %s' % (opsimGroup))
278 print(' OpsimRun = %s' % (opsimRun))
279 print(' OpsimComment = %s' % (opsimComment))
280 print(' OpsimVersion = %s' % (opsimVersion))
281 print(' OpsimDate = %s' % (opsimDate))
282 print(' MafVersion = %s' % (mafVersion))
283 print(' MafDate = %s' % (mafDate))
284 print(' Opsim dbFile = %s' % (dbFile))
285 runId = trackingDb.addRun(opsimGroup=opsimGroup, opsimRun=opsimRun, opsimComment=opsimComment,
286 opsimVersion=opsimVersion, opsimDate=opsimDate,
287 mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate,
288 mafDir=mafDir, dbFile=dbFile)
289 print('Used MAF RunID %d' % (runId))
290 trackingDb.close()