Coverage for python/lsst/sims/maf/db/trackingDb.py : 13%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import print_function
2from builtins import str
3from builtins import object
4import os
5from sqlalchemy import create_engine, Column, Integer, String
6from sqlalchemy.engine import url
7from sqlalchemy.orm import sessionmaker
9from sqlalchemy.ext.declarative import declarative_base
10from sqlalchemy.exc import DatabaseError
12Base = declarative_base()
14__all__ = ['TrackingDb', 'addRunToDatabase']
17class RunRow(Base):
18 """
19 Define contents and format of run list table.
21 Table to list all available MAF results, along with their opsim run and some comment info.
22 """
23 __tablename__ = "runs"
24 # Define columns in metric list table.
25 mafRunId = Column(Integer, primary_key=True)
26 opsimGroup = Column(String)
27 opsimRun = Column(String)
28 opsimComment = Column(String)
29 opsimVersion = Column(String)
30 opsimDate = Column(String)
31 dbFile = Column(String)
32 mafComment = Column(String)
33 mafVersion = Column(String)
34 mafDate = Column(String)
35 mafDir = Column(String)
36 def __repr__(self):
37 rstr = "<Run(mafRunId='%d', opsimGroup='%s', opsimRun='%s', opsimComment='%s', " \
38 "opsimVersion='%s', opsimDate='%s', mafComment='%s', " \
39 "mafVersion='%s', mafDate='%s', mafDir='%s', dbFile='%s'>" \
40 % (self.mafRunId, self.opsimGroup, self.opsimRun, self.opsimComment,
41 self.opsimVersion, self.opsimDate, self.mafComment, self.mafVersion, self.mafDate,
42 self.mafDir, self.dbFile)
43 return rstr
46class TrackingDb(object):
47 """Sqlite database to track MAF output runs and their locations, for showMaf.py
48 """
49 def __init__(self, database=None, trackingDbverbose=False):
50 """
51 Instantiate the results database, creating metrics, plots and summarystats tables.
52 """
53 self.verbose = trackingDbverbose
54 self.driver = 'sqlite'
55 # Connect to database
56 # for sqlite, connecting to non-existent database creates it automatically
57 if database is None:
58 # Default is a file in the current directory.
59 self.database = os.path.join(os.getcwd(), 'trackingDb_sqlite.db')
60 else:
61 self.database = database
62 # only sqlite
63 dbAddress = url.URL(drivername=self.driver, database=self.database)
64 engine = create_engine(dbAddress, echo=self.verbose)
65 if self.verbose:
66 print('Created or connected to MAF tracking %s database at %s' %(self.driver, self.database))
67 self.Session = sessionmaker(bind=engine)
68 self.session = self.Session()
69 # Create the tables, if they don't already exist.
70 try:
71 Base.metadata.create_all(engine)
72 except DatabaseError:
73 raise DatabaseError("Cannot create a %s database at %s. Check directory exists." %(self.driver,
74 self.database))
76 def close(self):
77 self.session.close()
79 def addRun(self, opsimGroup=None, opsimRun=None, opsimComment=None, opsimVersion=None, opsimDate=None,
80 mafComment=None, mafVersion=None, mafDate=None, mafDir=None, dbFile=None, mafRunId=None):
81 """Add a run to the tracking database.
83 Parameters
84 ----------
85 opsimGroup : str, opt
86 Set a name to group this run with (eg. "Tier 1, 2016").
87 opsimRun : str, opt
88 Set a name for the opsim run.
89 opsimComment : str, opt
90 Set a comment describing the opsim run.
91 opsimVersion : str, opt
92 Set the version of opsim.
93 opsimDate : str, opt
94 Set the date the opsim run was created.
95 mafComment : str, opt
96 Set a comment to describe the MAF analysis.
97 mafVersion : str, opt
98 Set the version of MAF used for analysis.
99 mafDate : str, opt
100 Set the date the MAF analysis was run.
101 mafDir : str, opt
102 The relative path to the MAF directory.
103 dbFile : str, opt
104 The relative path to the Opsim SQLite database file.
105 mafRunId : int, opt
106 The MafRunID to assign to this record in the database (note this is a primary key!).
107 If this run (ie the mafDir) exists in the database already, this will be ignored.
109 Returns
110 -------
111 int
112 The mafRunID stored in the database.
113 """
114 if opsimGroup is None:
115 opsimGroup = 'NULL'
116 if opsimRun is None:
117 opsimRun = 'NULL'
118 if opsimComment is None:
119 opsimComment = 'NULL'
120 if opsimVersion is None:
121 opsimVersion = 'NULL'
122 if opsimDate is None:
123 opsimDate = 'NULL'
124 if mafComment is None:
125 mafComment = 'NULL'
126 if mafVersion is None:
127 mafVersion = 'NULL'
128 if mafDate is None:
129 mafDate = 'NULL'
130 if mafDir is None:
131 mafDir = 'NULL'
132 if dbFile is None:
133 dbFile = 'NULL'
134 # Test if mafDir already exists in database.
135 prevrun = self.session.query(RunRow).filter_by(mafDir=mafDir).all()
136 if len(prevrun) > 0:
137 runIds = []
138 for run in prevrun:
139 runIds.append(run.mafRunId)
140 print('Updating information in tracking database - %s already present with runId %s.'
141 % (mafDir, runIds))
142 for run in prevrun:
143 self.session.delete(run)
144 self.session.commit()
145 runinfo = RunRow(mafRunId=runIds[0], opsimGroup=opsimGroup, opsimRun=opsimRun,
146 opsimComment=opsimComment, opsimVersion=opsimVersion, opsimDate=opsimDate,
147 mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate,
148 mafDir=mafDir, dbFile=dbFile)
149 else:
150 if mafRunId is not None:
151 # Check if mafRunId exists already.
152 existing = self.session.query(RunRow).filter_by(mafRunId=mafRunId).all()
153 if len(existing) > 0:
154 raise ValueError('MafRunId %d already exists in database, for %s. ' \
155 'Record must be deleted first.'
156 % (mafRunId, existing[0].mafDir))
157 runinfo = RunRow(mafRunId=mafRunId, opsimGroup=opsimGroup, opsimRun=opsimRun,
158 opsimComment=opsimComment, opsimVersion=opsimVersion, opsimDate=opsimDate,
159 mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate,
160 mafDir=mafDir, dbFile=dbFile)
161 else:
162 runinfo = RunRow(opsimGroup=opsimGroup, opsimRun=opsimRun,
163 opsimComment=opsimComment, opsimVersion=opsimVersion, opsimDate=opsimDate,
164 mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate,
165 mafDir=mafDir, dbFile=dbFile)
166 self.session.add(runinfo)
167 self.session.commit()
168 return runinfo.mafRunId
170 def delRun(self, runId):
171 """
172 Remove a run from the tracking database.
173 """
174 runinfo = self.session.query(RunRow).filter_by(mafRunId=runId).all()
175 if len(runinfo) == 0:
176 raise Exception('Could not find run with mafRunId %d' %(runId))
177 if len(runinfo) > 1:
178 raise Exception('Found more than one run with mafRunId %d' %(runId))
179 print('Removing run info for runId %d ' %(runId))
180 print(' ', runinfo)
181 self.session.delete(runinfo[0])
182 self.session.commit()
185def addRunToDatabase(mafDir, trackingDbFile, opsimGroup=None,
186 opsimRun=None, opsimComment=None,
187 mafComment=None, dbFile=None):
188 """Adds information about a MAF analysis run to a MAF tracking database.
190 Parameters
191 ----------
192 mafDir : str
193 Path to the directory where the MAF results are located.
194 trackingDb : str or lsst.sims.maf.TrackingDb
195 Full filename (+path) to the tracking database storing the MAF run information or
196 a TrackingDb object.
197 opsimGroup: str, opt
198 Name to use to group this run with other opsim runs. Default None.
199 opsimRun : str, opt
200 Name of the opsim run. If not provided, will attempt to use runName from confSummary.txt.
201 opsimComment : str, opt
202 Comment about the opsim run. If not provided, will attempt to use runComment from confSummary.txt.
203 mafComment : str, opt
204 Comment about the MAF analysis. If not provided, no comment will be recorded.
205 dbFile : str, opt
206 Relative path + name of the opsim database file. If not provided, no location will be recorded.
207 """
208 mafDir = os.path.abspath(mafDir)
209 if not os.path.isdir(mafDir):
210 raise ValueError('There is no directory containing MAF outputs at %s.' % (mafDir))
212 trackingDb = TrackingDb(database=trackingDbFile)
213 autoOpsimRun = None
214 autoOpsimComment = None
215 opsimVersion = None
216 opsimDate = None
217 mafVersion = None
218 mafDate = None
219 if os.path.isfile(os.path.join(mafDir, 'configSummary.txt')):
220 file = open(os.path.join(mafDir, 'configSummary.txt'))
221 for line in file:
222 tmp = line.split()
223 if tmp[0].startswith('RunName'):
224 autoOpsimRun = ' '.join(tmp[1:])
225 if tmp[0].startswith('RunComment'):
226 autoOpsimComment = ' '.join(tmp[1:])
227 # MAF Date may be in a line with "MafDate" (new configs)
228 # or at the end of "MAFVersion" (old configs).
229 if tmp[0].startswith('MAFDate'):
230 mafDate = tmp[-1]
231 if tmp[0].startswith('MAFVersion'):
232 mafVersion = tmp[1]
233 if len(tmp) > 2:
234 mafDate = tmp[-1]
235 if tmp[0].startswith('OpsimDate'):
236 opsimDate = tmp[-1]
237 if len(tmp) > 2:
238 opsimDate = tmp[-2]
239 if tmp[0].startswith('OpsimVersion'):
240 opsimVersion = tmp[1]
241 if len(tmp) > 2:
242 opsimDate = tmp[-2]
243 # And convert formats to '-' (again, multiple versions of configs).
244 if mafDate is not None:
245 if len(mafDate.split('/')) > 1:
246 t = mafDate.split('/')
247 if len(t[2]) == 2:
248 t[2] = '20' + t[2]
249 mafDate = '-'.join([t[2], t[1], t[0]])
250 if opsimDate is not None:
251 if len(opsimDate.split('/')) > 1:
252 t = opsimDate.split('/')
253 if len(t[2]) == 2:
254 t[2] = '20' + t[2]
255 opsimDate = '-'.join([t[2], t[1], t[0]])
257 if opsimRun is None:
258 opsimRun = autoOpsimRun
259 if opsimComment is None:
260 opsimComment = autoOpsimComment
262 print('Adding to tracking database at %s:' % (trackingDbFile))
263 print(' MafDir = %s' % (mafDir))
264 print(' MafComment = %s' % (mafComment))
265 print(' OpsimGroup = %s' % (opsimGroup))
266 print(' OpsimRun = %s' % (opsimRun))
267 print(' OpsimComment = %s' % (opsimComment))
268 print(' OpsimVersion = %s' % (opsimVersion))
269 print(' OpsimDate = %s' % (opsimDate))
270 print(' MafVersion = %s' % (mafVersion))
271 print(' MafDate = %s' % (mafDate))
272 print(' Opsim dbFile = %s' % (dbFile))
273 runId = trackingDb.addRun(opsimGroup=opsimGroup, opsimRun=opsimRun, opsimComment=opsimComment,
274 opsimVersion=opsimVersion, opsimDate=opsimDate,
275 mafComment=mafComment, mafVersion=mafVersion, mafDate=mafDate,
276 mafDir=mafDir, dbFile=dbFile)
277 print('Used MAF RunID %d' % (runId))
278 trackingDb.close()