Coverage for python/lsst/sims/maf/db/opsimDatabase.py : 5%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import print_function
2from builtins import str
3from builtins import zip
4import os, re
5import numpy as np
6import warnings
7from .database import Database
8from lsst.sims.utils import Site
9from lsst.sims.maf.utils import getDateVersion
11__all__ = ['testOpsimVersion', 'OpsimDatabase', 'OpsimDatabaseFBS', 'OpsimDatabaseV4', 'OpsimDatabaseV3']
13def testOpsimVersion(database, driver='sqlite', host=None, port=None):
14 opsdb = Database(database, driver=driver, host=host, port=port)
15 if 'SummaryAllProps' in opsdb.tableNames:
16 if 'Field' in opsdb.tableNames:
17 version = "V4"
18 else:
19 version = "FBS"
20 elif 'Summary' in opsdb.tableNames:
21 version = "V3"
22 else:
23 version = "Unknown"
24 opsdb.close()
25 return version
27def OpsimDatabase(database, driver='sqlite', host=None, port=None,
28 longstrings=False, verbose=False):
29 """Convenience method to return an appropriate OpsimDatabaseV3/V4 version.
31 This is here for backwards compatibility, as 'opsdb = db.OpsimDatabase(dbFile)' will
32 work as naively expected. However note that OpsimDatabase itself is no longer a class, but
33 a simple method that will attempt to instantiate the correct type of OpsimDatabaseV3 or OpsimDatabaseV4.
34 """
35 version = testOpsimVersion(database)
36 if version == 'FBS':
37 opsdb = OpsimDatabaseFBS(database, driver=driver, host=host, port=port,
38 longstrings=longstrings, verbose=verbose)
39 elif version == 'V4':
40 opsdb = OpsimDatabaseV4(database, driver=driver, host=host, port=port,
41 longstrings=longstrings, verbose=verbose)
42 elif version == 'V3':
43 opsdb = OpsimDatabaseV3(database, driver=driver, host=host, port=port,
44 longstrings=longstrings, verbose=verbose)
45 else:
46 warnings.warn('Could not identify opsim database version; just using Database class instead')
47 opsdb = Database(database, driver=driver, host=host, port=port,
48 longstrings=longstrings, verbose=verbose)
49 return opsdb
52class BaseOpsimDatabase(Database):
53 """Base opsim database class to gather common methods among different versions of the opsim schema.
55 Not intended to be used directly; use OpsimDatabaseFBS, OpsimDatabaseV3 or OpsimDatabaseV4 instead."""
56 def __init__(self, database, driver='sqlite', host=None, port=None, defaultTable=None,
57 longstrings=False, verbose=False):
58 super(BaseOpsimDatabase, self).__init__(database=database, driver=driver, host=host, port=port,
59 defaultTable=defaultTable, longstrings=longstrings,
60 verbose=verbose)
61 # Save filterlist so that we get the filter info per proposal in this desired order.
62 self.filterlist = np.array(['u', 'g', 'r', 'i', 'z', 'y'])
63 self.defaultTable = defaultTable
64 self._colNames()
66 def _colNames(self):
67 # Add version-specific column names in subclasses.
68 self.opsimVersion = 'unknown'
69 pass
71 def fetchMetricData(self, colnames, sqlconstraint=None, groupBy='default', tableName=None):
72 """
73 Fetch 'colnames' from 'tableName'.
75 Parameters
76 ----------
77 colnames : list
78 The columns to fetch from the table.
79 sqlconstraint : str, opt
80 The sql constraint to apply to the data (minus "WHERE"). Default None.
81 Examples: to fetch data for the r band filter only, set sqlconstraint to 'filter = "r"'.
82 groupBy : str, opt
83 The column to group the returned data by.
84 Default (when using summaryTable) is the MJD, otherwise will be None.
85 tableName : str, opt
86 The table to query. The default (None) will use the summary table, set by self.summaryTable.
88 Returns
89 -------
90 np.recarray
91 A structured array containing the data queried from the database.
92 """
93 if tableName is None:
94 tableName = self.defaultTable
95 if groupBy == 'default' and tableName==self.defaultTable:
96 groupBy = self.mjdCol
97 if groupBy == 'default' and tableName!=self.defaultTable:
98 groupBy = None
99 metricdata = super(BaseOpsimDatabase, self).fetchMetricData(colnames=colnames,
100 sqlconstraint=sqlconstraint,
101 groupBy=groupBy, tableName=tableName)
102 return metricdata
104 def fetchFieldsFromSummaryTable(self, sqlconstraint=None, raColName=None, decColName=None):
105 """
106 Fetch field information (fieldID/RA/Dec) from the summary table.
108 This implicitly only selects fields which were actually observed by opsim.
110 Parameters
111 ----------
112 sqlconstraint : str, opt
113 Sqlconstraint to apply before selecting observations to use for RA/Dec. Default None.
114 raColName : str, opt
115 Name of the RA column in the database.
116 decColName : str, opt
117 Name of the Dec column in the database.
118 degreesToRadians : bool, opt
119 Convert ra/dec into degrees?
120 If field information in summary table is in degrees, degreesToRadians should be True.
122 Returns
123 -------
124 np.recarray
125 Structured array containing the field data (fieldID, fieldRA, fieldDec). RA/Dec in radians.
126 """
127 if raColName is None:
128 raColName = self.raCol
129 if decColName is None:
130 decColName = self.decCol
131 fielddata = self.query_columns(self.defaultTable,
132 colnames=[self.fieldIdCol, raColName, decColName],
133 sqlconstraint=sqlconstraint, groupBy=self.fieldIdCol)
134 if self.raDecInDeg:
135 fielddata[raColName] = np.radians(fielddata[raColName])
136 fielddata[decColName] = np.radians(fielddata[decColName])
137 return fielddata
139 def fetchRunLength(self):
140 """Find the survey duration for a particular opsim run (years).
142 Returns
143 -------
144 float
145 """
146 if 'Config' not in self.tables:
147 print('Cannot access Config table to retrieve runLength; using default 10 years')
148 runLength = 10.0
149 else:
150 query = 'select paramValue from Config where paramName="%s"' % (self.runLengthParam)
151 runLength = self.query_arbitrary(query, dtype=[('paramValue', float)])
152 runLength = runLength['paramValue'][0] # Years
153 return runLength
155 def fetchLatLonHeight(self):
156 """Returns the latitude, longitude, and height of the telescope used by the config file.
157 """
158 if 'Config' not in self.tables:
159 print('Cannot access Config table to retrieve site parameters; using sims.utils.Site instead.')
160 site = Site(name='LSST')
161 lat = site.latitude_rad
162 lon = site.longitude_rad
163 height = site.height
164 else:
165 lat = self.query_columns('Config', colnames=['paramValue'],
166 sqlconstraint="paramName like '%latitude%'")
167 lat = float(lat['paramValue'][0])
168 lon = self.query_columns('Config', colnames=['paramValue'],
169 sqlconstraint="paramName like '%longitude%'")
170 lon = float(lon['paramValue'][0])
171 height = self.query_columns('Config', colnames=['paramValue'],
172 sqlconstraint="paramName like '%height%'")
173 height = float(height['paramValue'][0])
174 return lat, lon, height
176 def fetchOpsimRunName(self):
177 """Return opsim run name (machine name + session ID) from Session table.
178 """
179 if 'Session' not in self.tables:
180 print('Could not access Session table to find this information.')
181 runName = self.defaultRunName
182 else:
183 res = self.query_columns('Session', colnames=[self.sessionIdCol,
184 self.sessionHostCol])
185 runName = str(res[self.sessionHostCol][0]) + '_' + str(res[self.sessionIdCol][0])
186 return runName
188 def fetchNVisits(self, propId=None):
189 """Returns the total number of visits in the simulation or visits for a particular proposal.
191 Parameters
192 ----------
193 propId : int or list of ints
194 The ID numbers of the proposal(s).
196 Returns
197 -------
198 int
199 """
200 if 'ObsHistory' in self.tables and propId is None:
201 query = 'select count(*) from ObsHistory'
202 data = self.execute_arbitrary(query, dtype=([('nvisits', int)]))
203 else:
204 query = 'select count(distinct(%s)) from %s' %(self.mjdCol, self.defaultTable)
205 if propId is not None:
206 query += ' where '
207 if isinstance(propId, list) or isinstance(propId, np.ndarray):
208 for pID in propId:
209 query += 'propID=%d or ' %(int(pID))
210 query = query[:-3]
211 else:
212 query += 'propID = %d' %(int(propId))
213 data = self.execute_arbitrary(query, dtype=([('nvisits', int)]))
214 return data['nvisits'][0]
216 def fetchTotalSlewN(self):
217 """Return the total number of slews.
218 """
219 if 'SlewActivities' not in self.tables:
220 print('Could not access SlewActivities table to find this information.')
221 nslew = -1
222 else:
223 query = 'select count(distinct(%s)) from SlewActivities where %s >0' % (self.slewId,
224 self.delayCol)
225 res = self.execute_arbitrary(query, dtype=([('slewN', int)]))
226 nslew = res['slewN'][0]
227 return nslew
230class OpsimDatabaseFBS(BaseOpsimDatabase):
231 """
232 Database to class to interact with FBS versions of the opsim outputs.
234 Parameters
235 ----------
236 database : str
237 Name of the database or sqlite filename.
238 driver : str, opt
239 Name of the dialect + driver for sqlalchemy. Default 'sqlite'.
240 host : str, opt
241 Name of the database host. Default None (appropriate for sqlite files).
242 port : str, opt
243 String port number for the database. Default None (appropriate for sqlite files).
244 dbTables : dict, opt
245 Dictionary of the names of the tables in the database.
246 The dict should be key = table name, value = [table name, primary key].
247 """
248 def __init__(self, database, driver='sqlite', host=None, port=None, defaultTable='SummaryAllProps',
249 longstrings=False, verbose=False):
250 super().__init__(database=database, driver=driver, host=host, port=port,
251 defaultTable=defaultTable, longstrings=longstrings,
252 verbose=verbose)
254 def _colNames(self):
255 """
256 Set variables to represent the common column names used in this class directly.
258 This should make future schema changes a little easier to handle.
259 It is NOT meant to function as a general column map, just to abstract values
260 which are used *within this class*.
261 """
262 self.mjdCol = 'observationStartMJD'
263 self.raCol = 'fieldRA'
264 self.decCol = 'fieldDec'
265 self.propIdCol = 'proposalId'
266 # For config parsing.
267 self.versionCol = 'featureScheduler version'
268 self.dateCol = 'Date, ymd'
269 self.raDecInDeg = True
270 self.opsimVersion = 'FBS'
272 def fetchPropInfo(self):
273 """
274 There is no inherent proposal information or mapping in the FBS output.
275 An afterburner script does identify which visits may be counted as contributing toward WFD
276 and identifies these as such in the proposalID column (0 = general, 1 = WFD, 2+ = DD).
277 Returns dictionary of propID / propname, and dictionary of propTag / propID.
278 """
279 if 'Proposal' not in self.tables:
280 print('No proposal table available - no proposalIds have been assigned.')
281 return {}, {}
283 propIds = {}
284 # Add WFD and DD tags by default to propTags as we expect these every time. (avoids key errors).
285 propTags = {'WFD': [], 'DD': [], 'NES': []}
286 propData = self.query_columns('Proposal', colnames =[self.propIdCol, 'proposalName', 'proposalType'],
287 sqlconstraint=None)
288 for propId, propName, propType in zip(propData[self.propIdCol],
289 propData['proposalName'],
290 propData['proposalType']):
291 propIds[propId] = propName
292 if propType == 'WFD':
293 propTags['WFD'].append(propId)
294 if propType == 'DD':
295 propTags['DD'].append(propId)
296 return propIds, propTags
298 def createSQLWhere(self, tag, propTags):
299 """
300 Create a SQL constraint to identify observations taken for a particular proposal,
301 using the information in the propTags dictionary.
303 Parameters
304 ----------
305 tag : str
306 The name of the proposal for which to create a SQLwhere clause (WFD or DD).
307 propTags : dict
308 A dictionary of {proposal name : [proposal ids]}
309 This can be created using OpsimDatabase.fetchPropInfo()
311 Returns
312 -------
313 str
314 The SQL constraint, such as '(proposalID = 1) or (proposalID = 2)'
315 """
316 if (tag not in propTags) or (len(propTags[tag]) == 0):
317 print('No %s proposals found' % (tag))
318 # Create a sqlWhere clause that will not return anything as a query result.
319 sqlWhere = 'proposalId like "NO PROP"'
320 elif len(propTags[tag]) == 1:
321 sqlWhere = "proposalId = %d" % (propTags[tag][0])
322 else:
323 sqlWhere = "(" + " or ".join(["proposalId = %d" % (propid) for propid in propTags[tag]]) + ")"
324 return sqlWhere
326 def fetchConfig(self):
327 """
328 Fetch config data from configTable, match proposal IDs with proposal names and some field data,
329 and do a little manipulation of the data to make it easier to add to the presentation layer.
330 """
331 # Check to see if we're dealing with a full database or not. If not, just return (no config info to fetch).
332 if 'info' not in self.tables:
333 warnings.warn('Cannot fetch FBS config info.')
334 return {}, {}
335 # Create two dictionaries: a summary dict that contains a summary of the run
336 configSummary = {}
337 configSummary['keyorder'] = ['Version', 'RunInfo' ]
338 # and the other a general dict that contains all the details (by group) of the run.
339 config = {}
340 # Start to build up the summary.
341 # MAF version
342 mafdate, mafversion = getDateVersion()
343 configSummary['Version'] = {}
344 configSummary['Version']['MAFVersion'] = '%s' %(mafversion['__version__'])
345 configSummary['Version']['MAFDate'] = '%s' %(mafdate)
346 # Opsim date, version and runcomment info from config info table.
347 results = self.query_columns('info', ['Parameter', 'Value'],
348 sqlconstraint=f'Parameter like "{self.versionCol}"')
349 configSummary['Version']['OpsimVersion'] = 'FBS %s' %(results['Value'][0])
350 results = self.query_columns('info', ['Parameter', 'Value'],
351 sqlconstraint=f'Parameter like "{self.dateCol}"')
352 opsimdate = '-'.join(results['Value'][0].split(',')).replace(' ', '')
353 configSummary['Version']['OpsimDate'] = '%s' %(opsimdate)
354 configSummary['RunInfo'] = {}
355 results = self.query_columns('info', ['Parameter', 'Value'],
356 sqlconstraint=f'Parameter like "exec command%"')
357 configSummary['RunInfo']['Exec'] = '%s' % (results['Value'][0])
359 # Echo info table into configDetails.
360 configDetails = {}
361 configs = self.query_columns('info', ['Parameter', 'Value'])
362 for name, value in zip(configs['Parameter'], configs['Value']):
363 configDetails[name] = value
364 return configSummary, configDetails
367class OpsimDatabaseV4(BaseOpsimDatabase):
368 """
369 Database to class to interact with v4 versions of the opsim outputs.
371 Parameters
372 ----------
373 database : str
374 Name of the database or sqlite filename.
375 driver : str, opt
376 Name of the dialect + driver for sqlalchemy. Default 'sqlite'.
377 host : str, opt
378 Name of the database host. Default None (appropriate for sqlite files).
379 port : str, opt
380 String port number for the database. Default None (appropriate for sqlite files).
381 dbTables : dict, opt
382 Dictionary of the names of the tables in the database.
383 The dict should be key = table name, value = [table name, primary key].
384 """
385 def __init__(self, database, driver='sqlite', host=None, port=None, defaultTable='SummaryAllProps',
386 longstrings=False, verbose=False):
387 super(OpsimDatabaseV4, self).__init__(database=database, driver=driver, host=host, port=port,
388 defaultTable=defaultTable, longstrings=longstrings,
389 verbose=verbose)
391 def _colNames(self):
392 """
393 Set variables to represent the common column names used in this class directly.
395 This should make future schema changes a little easier to handle.
396 It is NOT meant to function as a general column map, just to abstract values
397 which are used *within this class*.
398 """
399 self.mjdCol = 'observationStartMJD'
400 self.slewId = 'slewHistory_slewCount'
401 self.delayCol = 'activityDelay'
402 self.fieldIdCol = 'fieldId'
403 self.raCol = 'fieldRA'
404 self.decCol = 'fieldDec'
405 self.propIdCol = 'propId'
406 self.propNameCol = 'propName'
407 self.propTypeCol = 'propType'
408 # For config parsing.
409 self.versionCol = 'version'
410 self.sessionIdCol = 'sessionId'
411 self.sessionHostCol = 'sessionHost'
412 self.sessionDateCol = 'sessionDate'
413 self.runCommentCol = 'runComment'
414 self.runLengthParam = 'survey/duration'
415 self.raDecInDeg = True
416 self.opsimVersion = 'V4'
418 def fetchFieldsFromFieldTable(self, propId=None, degreesToRadians=True):
419 """
420 Fetch field information (fieldID/RA/Dec) from the Field table.
422 This will select fields which were requested by a particular proposal or proposals,
423 even if they did not receive any observations.
425 Parameters
426 ----------
427 propId : int or list of ints
428 Proposal ID or list of proposal IDs to use to select fields.
429 degreesToRadians : bool, opt
430 If True, convert degrees in Field table into radians.
432 Returns
433 -------
434 np.recarray
435 Structured array containing the field data (fieldID, fieldRA, fieldDec).
436 """
437 if propId is not None:
438 query = 'select f.fieldId, f.ra, f.dec from Field as f'
439 query += ', ProposalField as p where (p.Field_fieldId = f.fieldId) '
440 if isinstance(propId, list) or isinstance(propId, np.ndarray):
441 query += ' and ('
442 for pID in propId:
443 query += '(p.Proposal_propId = %d) or ' % (int(pID))
444 # Remove the trailing 'or' and add a closing parenthesis.
445 query = query[:-3]
446 query += ')'
447 else: # single proposal ID.
448 query += ' and (p.Proposal_propId = %d) ' %(int(propId))
449 query += ' group by f.%s' %(self.fieldIdCol)
450 fielddata = self.query_arbitrary(query, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol],
451 ['int', 'float', 'float'])))
452 if len(fielddata) == 0:
453 fielddata = np.zeros(0, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol],
454 ['int', 'float', 'float'])))
455 else:
456 query = 'select fieldId, ra, dec from Field group by fieldId'
457 fielddata = self.query_arbitrary(query, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol],
458 ['int', 'float', 'float'])))
459 if len(fielddata) == 0:
460 fielddata = np.zeros(0, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol],
461 ['int', 'float', 'float'])))
462 if degreesToRadians:
463 fielddata[self.raCol] = fielddata[self.raCol] * np.pi / 180.
464 fielddata[self.decCol] = fielddata[self.decCol] * np.pi / 180.
465 return fielddata
468 def fetchPropInfo(self):
469 """
470 Fetch the proposal IDs as well as their (short) proposal names and science type tags from the
471 full opsim database.
472 Returns dictionary of propID / propname, and dictionary of propTag / propID.
473 If not using a full database, will return dict of propIDs with empty propnames + empty propTag dict.
474 """
475 propIds = {}
476 # Add WFD and DD tags by default to propTags as we expect these every time. (avoids key errors).
477 propTags = {'WFD': [], 'DD': [], 'NES': []}
478 assignData = True
479 try:
480 propData = self.query_columns('Proposal', colnames=[self.propIdCol, self.propNameCol],
481 sqlconstraint=None)
482 except ValueError:
483 propData = []
484 propIds = {}
485 propTags = {}
486 assignData = False
488 if assignData:
489 for propId, propName in zip(propData[self.propIdCol], propData[self.propNameCol]):
490 # Fix these in the future, to use the proper tags that will be added to output database.
491 propIds[propId] = propName
492 if 'widefastdeep' in propName.lower():
493 propTags['WFD'].append(propId)
494 if 'drilling' in propName.lower():
495 propTags['DD'].append(propId)
496 if 'northeclipticspur' in propName.lower():
497 propTags['NES'].append(propId)
498 return propIds, propTags
500 def createSlewConstraint(self, startTime=None, endTime=None):
501 """Create a SQL constraint for the slew tables (slew activities, slew speeds, slew states)
502 to select slews between startTime and endTime (MJD).
504 Parameters
505 ----------
506 startTime : float or None, opt
507 Time of first slew. Default (None) means no constraint on time of first slew.
508 endTime : float or None, opt
509 Time of last slew. Default (None) means no constraint on time of last slew.
511 Returns
512 -------
513 str
514 The SQL constraint, like 'slewHistory_slewID between XXX and XXXX'
515 """
516 query = 'select min(observationStartMJD), max(observationStartMJD) from obsHistory'
517 res = self.query_arbitrary(query, dtype=[('mjdMin', 'int'), ('mjdMax', 'int')])
518 # If asking for constraints that are out of bounds of the survey, return None.
519 if startTime is None or (startTime < res['mjdMin'][0]):
520 startTime = res['mjdMin'][0]
521 if endTime is None or (endTime > res['mjdMax'][0]):
522 endTime = res['mjdMax'][0]
523 # Check if times were out of bounds.
524 if startTime > res['mjdMax'][0] or endTime < res['mjdMin'][0]:
525 warnings.warn('Times requested do not overlap survey operations (%f to %f)'
526 % (res['mjdMin'][0], res['mjdMax'][0]))
527 return None
528 # Find the slew ID matching the start Time.
529 query = 'select min(observationId) from obsHistory where observationStartMJD >= %s' % (startTime)
530 res = self.query_arbitrary(query, dtype=[('observationId', 'int')])
531 obsHistId = res['observationId'][0]
532 query = 'select slewCount from slewHistory where ObsHistory_observationId = %d' % (obsHistId)
533 res = self.query_arbitrary(query, dtype=[('slewCount', 'int')])
534 minSlewCount = res['slewCount'][0]
535 # Find the slew ID matching the end Time.
536 query = 'select max(observationId) from obsHistory where observationStartMJD <= %s' % (endTime)
537 res = self.query_arbitrary(query, dtype=[('observationId', 'int')])
538 obsHistId = res['observationId'][0]
539 # Find the observation id corresponding to this slew time.
540 query = 'select slewCount from slewHistory where ObsHistory_observationId = %d' % (obsHistId)
541 res = self.query_arbitrary(query, dtype=[('slewCount', 'int')])
542 maxSlewCount = res['slewCount'][0]
543 return 'SlewHistory_slewCount between %d and %d' % (minSlewCount, maxSlewCount)
545 def createSQLWhere(self, tag, propTags):
546 """
547 Create a SQL constraint to identify observations taken for a particular proposal,
548 using the information in the propTags dictionary.
550 Parameters
551 ----------
552 tag : str
553 The name of the proposal for which to create a SQLwhere clause (WFD or DD).
554 propTags : dict
555 A dictionary of {proposal name : [proposal ids]}
556 This can be created using OpsimDatabase.fetchPropInfo()
558 Returns
559 -------
560 str
561 The SQL constraint, such as '(propID = 365) or (propID = 366)'
562 """
563 if (tag not in propTags) or (len(propTags[tag]) == 0):
564 print('No %s proposals found' % (tag))
565 # Create a sqlWhere clause that will not return anything as a query result.
566 sqlWhere = 'proposalId like "NO PROP"'
567 elif len(propTags[tag]) == 1:
568 sqlWhere = "proposalId = %d" % (propTags[tag][0])
569 else:
570 sqlWhere = "(" + " or ".join(["proposalId = %d" % (propid) for propid in propTags[tag]]) + ")"
571 return sqlWhere
573 def fetchRequestedNvisits(self, propId=None):
574 """Find the requested number of visits for the simulation or proposal(s).
576 Parameters
577 ----------
578 propId : int or list of ints, opt
580 Returns
581 -------
582 dict
583 Number of visits in u/g/r/i/z/y.
584 """
585 if propId is None:
586 constraint = ''
587 else:
588 if isinstance(propId, int):
589 constraint = 'propId = %d' % (propId)
590 else:
591 constraint = ''
592 for pId in propId:
593 constraint += 'propId = %d or ' % (int(pId))
594 constraint = constraint[:-3]
595 propData = self.query_columns('Proposal', colnames=[self.propNameCol, self.propTypeCol],
596 sqlconstraint=constraint)
597 nvisits = {}
598 for f in self.filterlist:
599 nvisits[f] = 0
600 for pName, propType in zip(propData[self.propNameCol], propData[self.propTypeCol]):
601 if propType.lower() == 'general':
602 for f in self.filterlist:
603 constraint = 'paramName="science/general_props/values/%s/filters/%s/num_visits"' \
604 % (pName, f)
605 val = self.query_columns('Config', colnames=['paramValue'], sqlconstraint=constraint)
606 if len(val) > 0:
607 nvisits[f] += int(val['paramValue'][0])
608 elif propType.lower == 'sequence':
609 pass
610 # Not clear yet.
611 return nvisits
613 def _matchParamNameValue(self, configarray, keyword):
614 return configarray['paramValue'][np.where(configarray['paramName'] == keyword)]
616 def _parseSequences(self, perPropConfig, filterlist):
617 """
618 (Private). Given an array of config paramName/paramValue info for a given WLTSS proposal
619 and the filterlist of filters used in that proposal, parse the sequences/subsequences and returns:
620 a dictionary with all the per-sequence information (including subsequence names & events, etc.)
621 a numpy array with the number of visits per filter
622 """
623 propDict = {}
624 # Identify where subsequences start in config[propname] arrays.
625 seqidxs = np.where(perPropConfig['paramName'] == 'SubSeqName')[0]
626 for sidx in seqidxs:
627 i = sidx
628 # Get the name of this subsequence.
629 seqname = perPropConfig['paramValue'][i]
630 # Check if seqname is a nested subseq of an existing sequence:
631 nestedsubseq = False
632 for prevseq in propDict:
633 if 'SubSeqNested' in propDict[prevseq]:
634 if seqname in propDict[prevseq]['SubSeqNested']:
635 seqdict = propDict[prevseq]['SubSeqNested'][seqname]
636 nestedsubseq = True
637 # If not, then create a new subseqence key/dictionary for this subseq.
638 if not nestedsubseq:
639 propDict[seqname] = {}
640 seqdict = propDict[seqname]
641 # And move on to next parameters within subsequence set.
642 i += 1
643 if perPropConfig['paramName'][i] == 'SubSeqNested':
644 subseqnestedname = perPropConfig['paramValue'][i]
645 if subseqnestedname != '.':
646 # Have nested subsequence, so keep track of that here
647 # but will fill in info later.
648 seqdict['SubSeqNested'] = {}
649 # Set up nested dictionary for nested subsequence.
650 seqdict['SubSeqNested'][subseqnestedname] = {}
651 i += 1
652 subseqfilters = perPropConfig['paramValue'][i]
653 if subseqfilters != '.':
654 seqdict['Filters'] = subseqfilters
655 i += 1
656 subseqexp = perPropConfig['paramValue'][i]
657 if subseqexp != '.':
658 seqdict['SubSeqExp'] = subseqexp
659 i+= 1
660 subseqevents = perPropConfig['paramValue'][i]
661 seqdict['Events'] = int(subseqevents)
662 i+=2
663 subseqinterval = perPropConfig['paramValue'][i]
664 subseqint = np.array([subseqinterval.split('*')], 'float').prod()
665 # In days ..
666 subseqint *= 1/24.0/60.0/60.0
667 if subseqint > 1:
668 seqdict['SubSeqInt'] = '%.2f days' %(subseqint)
669 else:
670 subseqint *= 24.0
671 if subseqint > 1:
672 seqdict['SubSeqInt'] = '%.2f hours' %(subseqint)
673 else:
674 subseqint *= 60.0
675 seqdict['SubSeqInt'] = '%.3f minutes' %(subseqint)
676 # End of assigning subsequence info - move on to counting number of visits.
677 nvisits = np.zeros(len(filterlist), int)
678 for subseq in propDict:
679 subevents = propDict[subseq]['Events']
680 # Count visits from direct subsequences.
681 if 'SubSeqExp' in propDict[subseq] and 'Filters' in propDict[subseq]:
682 subfilters = propDict[subseq]['Filters']
683 subexp = propDict[subseq]['SubSeqExp']
684 # If just one filter ..
685 if len(subfilters) == 1:
686 idx = np.where(filterlist == subfilters)[0]
687 nvisits[idx] += subevents * int(subexp)
688 else:
689 splitsubfilters = subfilters.split(',')
690 splitsubexp = subexp.split(',')
691 for f, exp in zip(splitsubfilters, splitsubexp):
692 idx = np.where(filterlist == f)[0]
693 nvisits[idx] += subevents * int(exp)
694 # Count visits if have nested subsequences.
695 if 'SubSeqNested' in propDict[subseq]:
696 for subseqnested in propDict[subseq]['SubSeqNested']:
697 events = subevents * propDict[subseq]['SubSeqNested'][subseqnested]['Events']
698 subfilters = propDict[subseq]['SubSeqNested'][subseqnested]['Filters']
699 subexp = propDict[subseq]['SubSeqNested'][subseqnested]['SubSeqExp']
700 # If just one filter ..
701 if len(subfilters) == 1:
702 idx = np.where(filterlist == subfilters)[0]
703 nvisits[idx] += events * int(subexp)
704 # Else may have multiple filters in the subsequence, so must split.
705 splitsubfilters = subfilters.split(',')
706 splitsubexp = subexp.split(',')
707 for f, exp in zip(splitsubfilters, splitsubexp):
708 idx = np.where(filterlist == f)[0]
709 nvisits[idx] += int(exp) * events
710 return propDict, nvisits
712 def _queryParam(self, constraint):
713 results = self.query_columns('Config', colnames=['paramValue'], sqlconstraint=constraint)
714 if len(results) > 0:
715 return results['paramValue'][0]
716 else:
717 return '--'
719 def fetchConfig(self):
720 """
721 Fetch config data from configTable, match proposal IDs with proposal names and some field data,
722 and do a little manipulation of the data to make it easier to add to the presentation layer.
723 """
724 # Check to see if we're dealing with a full database or not. If not, just return (no config info to fetch).
725 if 'Session' not in self.tables:
726 warnings.warn('Cannot fetch opsim config info as this is not a full opsim database.')
727 return {}, {}
728 # Create two dictionaries: a summary dict that contains a summary of the run
729 configSummary = {}
730 configSummary['keyorder'] = ['Version', 'RunInfo', 'Proposals']
731 # and the other a general dict that contains all the details (by group) of the run.
732 config = {}
733 # Start to build up the summary.
734 # MAF version
735 mafdate, mafversion = getDateVersion()
736 configSummary['Version'] = {}
737 configSummary['Version']['MAFVersion'] = '%s' %(mafversion['__version__'])
738 configSummary['Version']['MAFDate'] = '%s' %(mafdate)
739 # Opsim date, version and runcomment info from session table
740 results = self.query_columns('Session',
741 [self.versionCol, self.sessionDateCol, self.runCommentCol])
742 configSummary['Version']['OpsimVersion'] = '%s' %(results[self.versionCol][0])
743 configSummary['Version']['OpsimDate'] = '%s' %(results[self.sessionDateCol][0])
744 configSummary['Version']['OpsimDate'] = configSummary['Version']['OpsimDate'][0:10]
745 configSummary['RunInfo'] = {}
746 configSummary['RunInfo']['RunComment'] = results[self.runCommentCol][0]
747 configSummary['RunInfo']['RunName'] = self.fetchOpsimRunName()
748 # Pull out a few special values to put into summary.
749 # This section has a number of configuration parameter names hard-coded.
750 # I've left these here (rather than adding to self_colNames), bc I think schema changes in the config
751 # files will actually be easier to track here (at least until the opsim configs are cleaned up).
752 constraint = 'paramName="observatory/telescope/altitude_minpos"'
753 configSummary['RunInfo']['MinAlt'] = self._queryParam(constraint)
754 constraint = 'paramName="observatory/telescope/altitude_maxpos"'
755 configSummary['RunInfo']['MaxAlt'] = self._queryParam(constraint)
756 constraint = 'paramName="observatory/camera/filter_change_time"'
757 configSummary['RunInfo']['TimeFilterChange'] = self._queryParam(constraint)
758 constraint = 'paramName="observatory/camera/readout_time"'
759 configSummary['RunInfo']['TimeReadout'] = self._queryParam(constraint)
760 constraint = 'paramName="sched_driver/propboost_weight"'
761 configSummary['RunInfo']['PropBoostWeight'] = self._queryParam(constraint)
762 configSummary['RunInfo']['keyorder'] = ['RunName', 'RunComment', 'MinAlt', 'MaxAlt',
763 'TimeFilterChange', 'TimeReadout', 'PropBoostWeight']
765 # Echo config table into configDetails.
766 configDetails = {}
767 configs = self.query_columns('Config', ['paramName', 'paramValue'])
768 for name, value in zip(configs['paramName'], configs['paramValue']):
769 configDetails[name] = value
771 # Now finish building the summary to add proposal information.
772 # Loop through all proposals to add summary information.
773 propData = self.query_columns('Proposal', [self.propIdCol, self.propNameCol, self.propTypeCol])
774 configSummary['Proposals'] = {}
775 for propid, propname, proptype in zip(propData[self.propIdCol],
776 propData[self.propNameCol], propData[self.propTypeCol]):
777 configSummary['Proposals'][propname] = {}
778 propdict = configSummary['Proposals'][propname]
779 propdict['keyorder'] = ['PropId', 'PropName', 'PropType', 'Airmass bonus', 'Airmass max',
780 'HA bonus', 'HA max', 'Time weight', 'Restart Lost Sequences',
781 'Restart Complete Sequences', 'Filters']
782 propdict['PropName'] = propname
783 propdict['PropId'] = propid
784 propdict['PropType'] = proptype
785 # Add some useful information on the proposal parameters.
786 constraint = 'paramName like "science/%s_props/values/%s/sky_constraints/max_airmass"'\
787 % ("%", propname)
788 propdict['Airmass max'] = self._queryParam(constraint)
789 constraint = 'paramName like "science/%s_props/values/%s/scheduling/airmass_bonus"'\
790 % ("%", propname)
791 propdict['Airmass bonus'] = self._queryParam(constraint)
792 constraint = 'paramName like "science/%s_props/values/%s/scheduling/hour_angle_max"'\
793 % ("%", propname)
794 propdict['HA max'] = self._queryParam(constraint)
795 constraint = 'paramName like "science/%s_props/values/%s/scheduling/hour_angle_bonus"'\
796 % ("%", propname)
797 propdict['HA bonus'] = self._queryParam(constraint)
798 constraint = 'paramName like "science/%s_props/values/%s/scheduling/time_weight"'\
799 % ("%", propname)
800 propdict['Time weight'] = self._queryParam(constraint)
801 constraint = 'paramName like "science/%s_props/values/%s/scheduling/restart_lost_sequences"'\
802 % ("%", propname)
803 propdict['Restart Lost Sequences'] = self._queryParam(constraint)
804 constraint = 'paramName like "science/%s_props/values/%s/scheduling/restart_complete_sequences"'\
805 % ("%", propname)
806 propdict['Restart Complete Sequences'] = self._queryParam(constraint)
807 # Find number of visits requested per filter for the proposal
808 # along with min/max sky and airmass values.
809 propdict['Filters'] = {}
810 for f in self.filterlist:
811 propdict['Filters'][f] = {}
812 propdict['Filters'][f]['Filter'] = f
813 dictkeys = ['MaxSeeing', 'BrightLimit', 'DarkLimit', 'NumVisits', 'GroupedVisits', 'Snaps']
814 querykeys = ['max_seeing', 'bright_limit', 'dark_limit', 'num_visits',
815 'num_grouped_visits', 'exposures']
816 for dk, qk in zip(dictkeys, querykeys):
817 constraint = 'paramName like "science/%s_props/values/%s/filters/%s/%s"' \
818 % ("%", propname, f, qk)
819 propdict['Filters'][f][dk] = self._queryParam(constraint)
820 propdict['Filters'][f]['keyorder'] = ['Filter', 'MaxSeeing', 'MinSky', 'MaxSky',
821 'NumVisits', 'GroupedVisits', 'Snaps']
822 propdict['Filters']['keyorder'] = list(self.filterlist)
823 return configSummary, configDetails
826class OpsimDatabaseV3(BaseOpsimDatabase):
827 def __init__(self, database, driver='sqlite', host=None, port=None, defaultTable='Summary',
828 longstrings=False, verbose=False):
829 """
830 Instantiate object to handle queries of the opsim database.
831 (In general these will be the sqlite database files produced by opsim, but could
832 be any database holding those opsim output tables.).
834 database = Name of database or sqlite filename
835 driver = Name of database dialect+driver for sqlalchemy (e.g. 'sqlite', 'pymssql+mssql')
836 host = Name of database host (optional)
837 port = String port number (optional)
839 """
840 super(OpsimDatabaseV3, self).__init__(database=database, driver=driver, host=host, port=port,
841 defaultTable=defaultTable, longstrings=longstrings,
842 verbose=verbose)
844 def _colNames(self):
845 """
846 Set variables to represent the common column names used in this class directly.
848 This should make future schema changes a little easier to handle.
849 It is NOT meant to function as a general column map, just to abstract values
850 which are used *within this class*.
851 """
852 self.mjdCol = 'expMJD'
853 self.slewId = 'slewHistory_slewID'
854 self.delayCol = 'actDelay'
855 self.fieldIdCol = 'fieldID'
856 self.raCol = 'fieldRA'
857 self.decCol = 'fieldDec'
858 self.propIdCol = 'propID'
859 self.propConfCol = 'propConf'
860 self.propNameCol = 'propName' #(propname == proptype)
861 # For config parsing.
862 self.versionCol = 'version'
863 self.sessionIdCol = 'sessionID'
864 self.sessionHostCol = 'sessionHost'
865 self.sessionDateCol = 'sessionDate'
866 self.runCommentCol = 'runComment'
867 self.runLengthParam = 'nRun'
868 self.raDecInDeg = False
869 self.opsimVersion = 'V3'
871 def fetchFieldsFromFieldTable(self, propId=None, degreesToRadians=True):
872 """
873 Fetch field information (fieldID/RA/Dec) from Field (+Proposal_Field) tables.
875 propID = the proposal ID (default None), if selecting particular proposal - can be a list
876 degreesToRadians = RA/Dec values are in degrees in the Field table (so convert to radians).
877 """
878 # Note that you can't select any other sql constraints (such as filter).
879 # This will select fields which were requested by a particular proposal or proposals,
880 # even if they didn't get any observations.
881 if propId is not None:
882 query = 'select f.%s, f.%s, f.%s from %s as f' %(self.fieldIdCol, self.raCol, self.decCol,
883 'Field')
884 query += ', %s as p where (p.Field_%s = f.%s) ' %('Proposal_Field',
885 self.fieldIdCol, self.fieldIdCol)
886 if isinstance(propId, list) or isinstance(propId, np.ndarray):
887 query += ' and ('
888 for pID in propId:
889 query += '(p.Proposal_%s = %d) or ' %(self.propIdCol, int(pID))
890 # Remove the trailing 'or' and add a closing parenthesis.
891 query = query[:-3]
892 query += ')'
893 else: # single proposal ID.
894 query += ' and (p.Proposal_%s = %d) ' %(self.propIdCol, int(propId))
895 query += ' group by f.%s' %(self.fieldIdCol)
896 fielddata = self.query_arbitrary(query, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol],
897 ['int', 'float', 'float'])))
898 if len(fielddata) == 0:
899 fielddata = np.zeros(0, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol],
900 ['int', 'float', 'float'])))
901 else:
902 fielddata = self.query_columns('Field', colnames=[self.fieldIdCol, self.raCol, self.decCol],
903 groupBy = self.fieldIdCol)
904 if degreesToRadians:
905 fielddata[self.raCol] = fielddata[self.raCol] * np.pi / 180.
906 fielddata[self.decCol] = fielddata[self.decCol] * np.pi / 180.
907 return fielddata
909 def fetchPropInfo(self):
910 """
911 Fetch the proposal IDs as well as their (short) proposal names and science type tags from the
912 full opsim database.
913 Returns dictionary of propID / propname, and dictionary of propTag / propID.
914 If not using a full database, will return dict of propIDs with empty propnames + empty propTag dict.
915 """
916 propIDs = {}
917 # Add WFD and DD tags by default to propTags as we expect these every time. (avoids key errors).
918 propTags = {'WFD':[], 'DD':[], 'NES': []}
919 # If do not have full database available:
920 if 'Proposal' not in self.tables:
921 propData = self.query_columns(self.defaultTable, colnames=[self.propIdCol])
922 for propid in propData[self.propIdCol]:
923 propIDs[int(propid)] = propid
924 else:
925 # Query for all propIDs.
926 propData = self.query_columns('Proposal', colnames=[self.propIdCol, self.propConfCol,
927 self.propNameCol], sqlconstraint=None)
928 for propid, propname in zip(propData[self.propIdCol], propData[self.propConfCol]):
929 # Strip '.conf', 'Prop', and path info.
930 propIDs[int(propid)] = re.sub('Prop','', re.sub('.conf','', re.sub('.*/', '', propname)))
931 # Find the 'ScienceType' from the config table, to indicate DD/WFD/Rolling, etc.
932 sciencetypes = self.query_columns('Config', colnames=['paramValue', 'nonPropID'],
933 sqlconstraint="paramName like 'ScienceType'")
934 if len(sciencetypes) == 0:
935 # Then this was an older opsim run without 'ScienceType' tags,
936 # so fall back to trying to guess what proposals are WFD or DD.
937 for propid, propname in propIDs.items():
938 if 'universal' in propname.lower():
939 propTags['WFD'].append(propid)
940 if 'deep' in propname.lower():
941 propTags['DD'].append(propid)
942 if 'northecliptic' in propname.lower():
943 propTags['NES'].append(propid)
944 else:
945 # Newer opsim output with 'ScienceType' fields in conf files.
946 for sc in sciencetypes:
947 # ScienceType tag can be multiple values, separated by a ','
948 tags = [x.strip(' ') for x in sc['paramValue'].split(',')]
949 for sciencetype in tags:
950 if sciencetype in propTags:
951 propTags[sciencetype].append(int(sc['nonPropID']))
952 else:
953 propTags[sciencetype] = [int(sc['nonPropID']),]
954 # But even these runs don't tag NES
955 for propid, propname in propIDs.items():
956 if 'northecliptic' in propname.lower():
957 propTags['NES'].append(propid)
958 return propIDs, propTags
960 def createSlewConstraint(self, startTime=None, endTime=None):
961 """Create a SQL constraint for the slew tables (slew activities, slew speeds, slew states)
962 to select slews between startTime and endTime (MJD).
964 Parameters
965 ----------
966 startTime : float or None, opt
967 Time of first slew. Default (None) means no constraint on time of first slew.
968 endTime : float or None, opt
969 Time of last slew. Default (None) means no constraint on time of last slew.
971 Returns
972 -------
973 str
974 The SQL constraint, like 'slewHistory_slewID > XXX and slewHistory_slewID < XXXX'
975 """
976 query = 'select min(expMJD), max(expMJD) from obsHistory'
977 res = self.query_arbitrary(query, dtype=[('mjdMin', 'int'), ('mjdMax', 'int')])
978 # If asking for constraints that are out of bounds of the survey, return None.
979 if startTime is None or (startTime < res['mjdMin'][0]):
980 startTime = res['mjdMin'][0]
981 if endTime is None or (endTime > res['mjdMax'][0]):
982 endTime = res['mjdMax'][0]
983 # Check if times were out of bounds.
984 if startTime > res['mjdMax'][0] or endTime < res['mjdMin'][0]:
985 warnings.warn('Times requested do not overlap survey operations (%f to %f)'
986 % (res['mjdMin'][0], res['mjdMax'][0]))
987 return None
988 # Find the slew ID matching the start Time.
989 query = 'select min(obsHistID) from obsHistory where expMJD >= %s' % (startTime)
990 res = self.query_arbitrary(query, dtype=[('observationId', 'int')])
991 obsHistId = res['observationId'][0]
992 query = 'select slewID from slewHistory where ObsHistory_obsHistID = %d' % (obsHistId)
993 res = self.query_arbitrary(query, dtype=[('slewCount', 'int')])
994 minSlewCount = res['slewCount'][0]
995 # Find the slew ID matching the end Time.
996 query = 'select max(obsHistID) from obsHistory where expMJD <= %s' % (endTime)
997 res = self.query_arbitrary(query, dtype=[('observationId', 'int')])
998 obsHistId = res['observationId'][0]
999 # Find the observation id corresponding to this slew time.
1000 query = 'select slewID from slewHistory where ObsHistory_obsHistID = %d' % (obsHistId)
1001 res = self.query_arbitrary(query, dtype=[('slewCount', 'int')])
1002 maxSlewCount = res['slewCount'][0]
1003 return 'SlewHistory_slewID between %d and %d' % (minSlewCount, maxSlewCount)
1005 def createSQLWhere(self, tag, propTags):
1006 """Create a SQL constraint to identify observations taken for a particular proposal,
1007 using the information in the propTags dictionary.
1009 Parameters
1010 ----------
1011 tag : str
1012 The name of the proposal for which to create a SQLwhere clause (WFD or DD).
1013 propTags : dict
1014 A dictionary of {proposal name : [proposal ids]}
1015 This can be created using OpsimDatabase.fetchPropInfo()
1017 Returns
1018 -------
1019 str
1020 The SQL constraint, such as '(propID = 365) or (propID = 366)'
1021 """
1022 if (tag not in propTags) or (len(propTags[tag]) == 0):
1023 print('No %s proposals found' % (tag))
1024 # Create a sqlWhere clause that will not return anything as a query result.
1025 sqlWhere = 'propID like "NO PROP"'
1026 elif len(propTags[tag]) == 1:
1027 sqlWhere = "propID = %d" % (propTags[tag][0])
1028 else:
1029 sqlWhere = "(" + " or ".join(["propID = %d" % (propid) for propid in propTags[tag]]) + ")"
1030 return sqlWhere
1032 def fetchRequestedNvisits(self, propId=None):
1033 """
1034 Find the requested number of visits for proposals in propId.
1035 Returns a dictionary - Nvisits{u/g/r/i/z/y}
1036 """
1037 visitDict = {}
1038 if propId is None:
1039 # Get all the available propIds.
1040 propData = self.query_columns('Proposal', colnames=[self.propIdCol, self.propNameCol],
1041 sqlconstraint=None)
1042 else:
1043 # Get the propType info to go with the propId(s).
1044 if hasattr(propId, '__iter__'):
1045 constraint = '('
1046 for pi in propId:
1047 constraint += '(propId = %d) or ' %(pi)
1048 constraint = constraint[:-4] + ')'
1049 else:
1050 constraint = 'propId = %d' %(propId)
1051 propData = self.query_columns('Proposal', colnames=[self.propIdCol, self.propNameCol],
1052 sqlconstraint=constraint)
1053 for pId, propType in zip(propData[self.propIdCol], propData[self.propNameCol]):
1054 perPropConfig = self.query_columns('Config', colnames=['paramName', 'paramValue'],
1055 sqlconstraint = 'nonPropID = %d and paramName!="userRegion"'
1056 %(pId))
1057 filterlist = self._matchParamNameValue(perPropConfig, 'Filter')
1058 if propType == 'WL':
1059 # For WL proposals, the simple 'Filter_Visits' == the requested number of observations.
1060 nvisits = np.array(self._matchParamNameValue(perPropConfig, 'Filter_Visits'), int)
1061 elif propType == 'WLTSS':
1062 seqDict, nvisits = self._parseSequences(perPropConfig, filterlist)
1063 visitDict[pId] = {}
1064 for f, N in zip(filterlist, nvisits):
1065 visitDict[pId][f] = N
1066 nvisits = {}
1067 for f in ['u', 'g', 'r', 'i', 'z', 'y']:
1068 nvisits[f] = 0
1069 for pId in visitDict:
1070 for f in visitDict[pId]:
1071 nvisits[f] += visitDict[pId][f]
1072 return nvisits
1074 def _matchParamNameValue(self, configarray, keyword):
1075 return configarray['paramValue'][np.where(configarray['paramName']==keyword)]
1077 def _parseSequences(self, perPropConfig, filterlist):
1078 """
1079 (Private). Given an array of config paramName/paramValue info for a given WLTSS proposal
1080 and the filterlist of filters used in that proposal, parse the sequences/subsequences and returns:
1081 a dictionary with all the per-sequence information (including subsequence names & events, etc.)
1082 a numpy array with the number of visits per filter
1083 """
1084 propDict = {}
1085 # Identify where subsequences start in config[propname] arrays.
1086 seqidxs = np.where(perPropConfig['paramName'] == 'SubSeqName')[0]
1087 for sidx in seqidxs:
1088 i = sidx
1089 # Get the name of this subsequence.
1090 seqname = perPropConfig['paramValue'][i]
1091 # Check if seqname is a nested subseq of an existing sequence:
1092 nestedsubseq = False
1093 for prevseq in propDict:
1094 if 'SubSeqNested' in propDict[prevseq]:
1095 if seqname in propDict[prevseq]['SubSeqNested']:
1096 seqdict = propDict[prevseq]['SubSeqNested'][seqname]
1097 nestedsubseq = True
1098 # If not, then create a new subseqence key/dictionary for this subseq.
1099 if not nestedsubseq:
1100 propDict[seqname] = {}
1101 seqdict = propDict[seqname]
1102 # And move on to next parameters within subsequence set.
1103 i += 1
1104 if perPropConfig['paramName'][i] == 'SubSeqNested':
1105 subseqnestedname = perPropConfig['paramValue'][i]
1106 if subseqnestedname != '.':
1107 # Have nested subsequence, so keep track of that here
1108 # but will fill in info later.
1109 seqdict['SubSeqNested'] = {}
1110 # Set up nested dictionary for nested subsequence.
1111 seqdict['SubSeqNested'][subseqnestedname] = {}
1112 i += 1
1113 subseqfilters = perPropConfig['paramValue'][i]
1114 if subseqfilters != '.':
1115 seqdict['Filters'] = subseqfilters
1116 i += 1
1117 subseqexp = perPropConfig['paramValue'][i]
1118 if subseqexp != '.':
1119 seqdict['SubSeqExp'] = subseqexp
1120 i+= 1
1121 subseqevents = perPropConfig['paramValue'][i]
1122 seqdict['Events'] = int(subseqevents)
1123 i+=2
1124 subseqinterval = perPropConfig['paramValue'][i]
1125 subseqint = np.array([subseqinterval.split('*')], 'float').prod()
1126 # In days ..
1127 subseqint *= 1/24.0/60.0/60.0
1128 if subseqint > 1:
1129 seqdict['SubSeqInt'] = '%.2f days' %(subseqint)
1130 else:
1131 subseqint *= 24.0
1132 if subseqint > 1:
1133 seqdict['SubSeqInt'] = '%.2f hours' %(subseqint)
1134 else:
1135 subseqint *= 60.0
1136 seqdict['SubSeqInt'] = '%.3f minutes' %(subseqint)
1137 # End of assigning subsequence info - move on to counting number of visits.
1138 nvisits = np.zeros(len(filterlist), int)
1139 for subseq in propDict:
1140 subevents = propDict[subseq]['Events']
1141 # Count visits from direct subsequences.
1142 if 'SubSeqExp' in propDict[subseq] and 'Filters' in propDict[subseq]:
1143 subfilters = propDict[subseq]['Filters']
1144 subexp = propDict[subseq]['SubSeqExp']
1145 # If just one filter ..
1146 if len(subfilters) == 1:
1147 idx = np.where(filterlist == subfilters)[0]
1148 nvisits[idx] += subevents * int(subexp)
1149 else:
1150 splitsubfilters = subfilters.split(',')
1151 splitsubexp = subexp.split(',')
1152 for f, exp in zip(splitsubfilters, splitsubexp):
1153 idx = np.where(filterlist == f)[0]
1154 nvisits[idx] += subevents * int(exp)
1155 # Count visits if have nested subsequences.
1156 if 'SubSeqNested' in propDict[subseq]:
1157 for subseqnested in propDict[subseq]['SubSeqNested']:
1158 events = subevents * propDict[subseq]['SubSeqNested'][subseqnested]['Events']
1159 subfilters = propDict[subseq]['SubSeqNested'][subseqnested]['Filters']
1160 subexp = propDict[subseq]['SubSeqNested'][subseqnested]['SubSeqExp']
1161 # If just one filter ..
1162 if len(subfilters) == 1:
1163 idx = np.where(filterlist == subfilters)[0]
1164 nvisits[idx] += events * int(subexp)
1165 # Else may have multiple filters in the subsequence, so must split.
1166 splitsubfilters = subfilters.split(',')
1167 splitsubexp = subexp.split(',')
1168 for f, exp in zip(splitsubfilters, splitsubexp):
1169 idx = np.where(filterlist == f)[0]
1170 nvisits[idx] += int(exp) * events
1171 return propDict, nvisits
1173 def fetchConfig(self):
1174 """
1175 Fetch config data from configTable, match proposal IDs with proposal names and some field data,
1176 and do a little manipulation of the data to make it easier to add to the presentation layer.
1177 """
1178 # Check to see if we're dealing with a full database or not. If not, just return (no config info to fetch).
1179 if 'Session' not in self.tables:
1180 warnings.warn('Cannot fetch opsim config info as this is not a full opsim database.')
1181 return {}, {}
1182 # Create two dictionaries: a summary dict that contains a summary of the run
1183 configSummary = {}
1184 configSummary['keyorder'] = ['Version', 'RunInfo', 'Proposals']
1185 # and the other a general dict that contains all the details (by group) of the run.
1186 config = {}
1187 # Start to build up the summary.
1188 # MAF version
1189 mafdate, mafversion = getDateVersion()
1190 configSummary['Version'] = {}
1191 configSummary['Version']['MAFVersion'] = '%s' % (mafversion['__version__'])
1192 configSummary['Version']['MAFDate'] = '%s' % (mafdate)
1193 # Opsim date, version and runcomment info from session table
1194 results = self.query_columns('Session', colnames = [self.versionCol, self.sessionDateCol,
1195 self.runCommentCol])
1196 configSummary['Version']['OpsimVersion'] = '%s' % (results[self.versionCol][0])
1197 configSummary['Version']['OpsimDate'] = '%s' % (results[self.sessionDateCol][0])
1198 configSummary['RunInfo'] = {}
1199 configSummary['RunInfo']['RunComment'] = results[self.runCommentCol]
1200 configSummary['RunInfo']['RunName'] = self.fetchOpsimRunName()
1201 # Pull out a few special values to put into summary.
1202 # This section has a number of configuration parameter names hard-coded.
1203 # I've left these here (rather than adding to self_colNames), because I think schema changes will in the config
1204 # files will actually be easier to track here (at least until the opsim configs are cleaned up).
1205 constraint = 'moduleName="Config" and paramName="sha1"'
1206 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint)
1207 try:
1208 configSummary['Version']['Config sha1'] = results['paramValue'][0]
1209 except IndexError:
1210 configSummary['Version']['Config sha1'] = 'Unavailable'
1211 constraint = 'moduleName="Config" and paramName="changedFiles"'
1212 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint)
1213 try:
1214 configSummary['Version']['Config changed files'] = results['paramValue'][0]
1215 except IndexError:
1216 configSummary['Version']['Config changed files'] = 'Unavailable'
1217 constraint = 'moduleName="instrument" and paramName="Telescope_AltMin"'
1218 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint)
1219 configSummary['RunInfo']['MinAlt'] = results['paramValue'][0]
1220 constraint = 'moduleName="instrument" and paramName="Telescope_AltMax"'
1221 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint)
1222 configSummary['RunInfo']['MaxAlt'] = results['paramValue'][0]
1223 constraint = 'moduleName="instrument" and paramName="Filter_MoveTime"'
1224 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint)
1225 configSummary['RunInfo']['TimeFilterChange'] = results['paramValue'][0]
1226 constraint = 'moduleName="instrument" and paramName="Readout_Time"'
1227 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint)
1228 configSummary['RunInfo']['TimeReadout'] = results['paramValue'][0]
1229 constraint = 'moduleName="scheduler" and paramName="MinDistance2Moon"'
1230 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint)
1231 configSummary['RunInfo']['MinDist2Moon'] = results['paramValue'][0]
1232 configSummary['RunInfo']['keyorder'] = ['RunName', 'RunComment', 'MinDist2Moon', 'MinAlt', 'MaxAlt',
1233 'TimeFilterChange', 'TimeReadout']
1234 # Now build up config dict with 'nice' group names (proposal name and short module name)
1235 # Each dict entry is a numpy array with the paramName/paramValue/comment values.
1236 # Match proposal IDs with names.
1237 query = 'select %s, %s, %s from Proposal group by %s' %(self.propIdCol, self.propConfCol,
1238 self.propNameCol, self.propIdCol)
1239 propdata = self.query_arbitrary(query, dtype=([(self.propIdCol, int),
1240 (self.propConfCol, np.str_, 256),
1241 (self.propNameCol, np.str_, 256)]))
1242 # Make 'nice' proposal names
1243 propnames = np.array([os.path.split(x)[1].replace('.conf', '') for x in propdata[self.propConfCol]])
1244 # Get 'nice' module names
1245 moduledata = self.query_columns('Config', colnames=['moduleName',], sqlconstraint='nonPropID=0')
1246 modulenames = np.array([os.path.split(x)[1].replace('.conf', '') for x in moduledata['moduleName']])
1247 # Grab the config information for each proposal and module.
1248 cols = ['paramName', 'paramValue', 'comment']
1249 for longmodname, modname in zip(moduledata['moduleName'], modulenames):
1250 config[modname] = self.query_columns('Config', colnames=cols,
1251 sqlconstraint='moduleName="%s"' %(longmodname))
1252 config[modname] = config[modname][['paramName', 'paramValue', 'comment']]
1253 for propid, propname in zip(propdata[self.propIdCol], propnames):
1254 config[propname] = self.query_columns('Config', colnames=cols,
1255 sqlconstraint='nonPropID="%s" and paramName!="userRegion"'
1256 %(propid))
1257 config[propname] = config[propname][['paramName', 'paramValue', 'comment']]
1258 config['keyorder'] = ['Comment', 'LSST', 'site', 'instrument', 'filters',
1259 'AstronomicalSky', 'File', 'scheduler',
1260 'schedulingData', 'schedDown', 'unschedDown']
1261 # Now finish building the summary to add proposal information.
1262 # Loop through all proposals to add summary information.
1263 configSummary['Proposals'] = {}
1264 propidorder = sorted(propdata[self.propIdCol])
1265 # Generate a keyorder to print proposals in order of propid.
1266 configSummary['Proposals']['keyorder'] = []
1267 for propid in propidorder:
1268 configSummary['Proposals']['keyorder'].append(propnames[np.where(propdata[self.propIdCol]
1269 == propid)][0])
1270 for propid, propname in zip(propdata[self.propIdCol], propnames):
1271 configSummary['Proposals'][propname] = {}
1272 propdict = configSummary['Proposals'][propname]
1273 propdict['keyorder'] = [self.propIdCol, self.propNameCol, 'PropType', 'RelPriority',
1274 'NumUserRegions', 'NumFields']
1275 propdict[self.propNameCol] = propname
1276 propdict[self.propIdCol] = propid
1277 propdict['PropType'] = propdata[self.propNameCol][np.where(propnames == propname)]
1278 propdict['RelPriority'] = self._matchParamNameValue(config[propname], 'RelativeProposalPriority')
1279 # Get the number of user regions.
1280 constraint = 'nonPropID="%s" and paramName="userRegion"' %(propid)
1281 result = self.query_columns('Config', colnames=['paramName',], sqlconstraint=constraint)
1282 propdict['NumUserRegions'] = result.size
1283 # Get the number of fields requested in the proposal (all filters).
1284 propdict['NumFields'] = self.fetchFieldsFromFieldTable(propId=propid).size
1285 # Find number of visits requested per filter for the proposal, with min/max sky & airmass values.
1286 # Note that config table has multiple entries for Filter/Filter_Visits/etc. with the same name.
1287 # The order of these entries in the config array matters.
1288 propdict['PerFilter'] = {}
1289 for key, keyword in zip(['Filters', 'MaxSeeing', 'MinSky', 'MaxSky'],
1290 ['Filter', 'Filter_MaxSeeing', 'Filter_MinBrig', 'Filter_MaxBrig']):
1291 temp = self._matchParamNameValue(config[propname], keyword)
1292 if len(temp) > 0:
1293 propdict['PerFilter'][key] = temp
1294 # Add exposure time, potentially looking for scaling per filter.
1295 exptime = float(self._matchParamNameValue(config[propname], 'ExposureTime')[0])
1296 temp = self._matchParamNameValue(config[propname], 'Filter_ExpFactor')
1297 if len(temp) > 0:
1298 propdict['PerFilter']['VisitTime'] = temp * exptime
1299 else:
1300 propdict['PerFilter']['VisitTime'] = np.ones(len(propdict['PerFilter']['Filters']), float)
1301 propdict['PerFilter']['VisitTime'] *= exptime
1302 # And count how many total exposures are requested per filter.
1303 # First check if 'RestartCompleteSequences' is true:
1304 # if both are true, then basically an indefinite number of visits are requested, although we're
1305 # not going to count this way (as the proposals still make an approximate number of requests).
1306 restartComplete = False
1307 temp = self._matchParamNameValue(config[propname], 'RestartCompleteSequences')
1308 if len(temp) > 0:
1309 if temp[0] == 'True':
1310 restartComplete = True
1311 propdict['RestartCompleteSequences'] = restartComplete
1312 # Grab information on restarting lost sequences so we can print this too.
1313 restartLost = False
1314 tmp = self._matchParamNameValue(config[propname], 'RestartLostSequences')
1315 if len(temp) > 0:
1316 if temp[0] == 'True':
1317 restartLost = True
1318 propdict['RestartLostSequences'] = restartLost
1319 if propdict['PropType'] == 'WL':
1320 # Simple 'Filter_Visits' request for number of observations.
1321 propdict['PerFilter']['NumVisits'] = np.array(self._matchParamNameValue(config[propname],
1322 'Filter_Visits'), int)
1323 elif propdict['PropType'] == 'WLTSS':
1324 # Proposal contains subsequences and possible nested subseq, so must delve further.
1325 # Make a dictionary to hold the subsequence info (keyed per subsequence).
1326 propdict['SubSeq'], Nvisits = self._parseSequences(config[propname],
1327 propdict['PerFilter']['Filters'])
1328 propdict['PerFilter']['NumVisits'] = Nvisits
1329 propdict['SubSeq']['keyorder'] = ['SubSeqName', 'SubSeqNested', 'Events', 'SubSeqInt']
1330 # Sort the filter information so it's ugrizy instead of order in opsim config db.
1331 idx = []
1332 for f in self.filterlist:
1333 filterpos = np.where(propdict['PerFilter']['Filters'] == f)
1334 if len(filterpos[0]) > 0:
1335 idx.append(filterpos[0][0])
1336 idx = np.array(idx, int)
1337 for k in propdict['PerFilter']:
1338 tmp = propdict['PerFilter'][k][idx]
1339 propdict['PerFilter'][k] = tmp
1340 propdict['PerFilter']['keyorder'] = ['Filters', 'VisitTime', 'MaxSeeing', 'MinSky',
1341 'MaxSky', 'NumVisits']
1342 return configSummary, config