Coverage for python/lsst/sims/maf/db/opsimDatabase.py : 5%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import print_function
2from builtins import str
3from builtins import zip
4import os, re
5import numpy as np
6import warnings
7from .database import Database
8from lsst.sims.utils import Site
9from lsst.sims.maf.utils import getDateVersion
10from sqlalchemy import column
12__all__ = ['testOpsimVersion', 'OpsimDatabase', 'OpsimDatabaseFBS', 'OpsimDatabaseV4', 'OpsimDatabaseV3']
14def testOpsimVersion(database, driver='sqlite', host=None, port=None):
15 opsdb = Database(database, driver=driver, host=host, port=port)
16 if 'SummaryAllProps' in opsdb.tableNames:
17 if 'Field' in opsdb.tableNames:
18 version = "V4"
19 else:
20 version = "FBS"
21 elif 'Summary' in opsdb.tableNames:
22 version = "V3"
23 else:
24 version = "Unknown"
25 opsdb.close()
26 return version
28def OpsimDatabase(database, driver='sqlite', host=None, port=None,
29 longstrings=False, verbose=False):
30 """Convenience method to return an appropriate OpsimDatabaseV3/V4 version.
32 This is here for backwards compatibility, as 'opsdb = db.OpsimDatabase(dbFile)' will
33 work as naively expected. However note that OpsimDatabase itself is no longer a class, but
34 a simple method that will attempt to instantiate the correct type of OpsimDatabaseV3 or OpsimDatabaseV4.
35 """
36 version = testOpsimVersion(database)
37 if version == 'FBS':
38 opsdb = OpsimDatabaseFBS(database, driver=driver, host=host, port=port,
39 longstrings=longstrings, verbose=verbose)
40 elif version == 'V4':
41 opsdb = OpsimDatabaseV4(database, driver=driver, host=host, port=port,
42 longstrings=longstrings, verbose=verbose)
43 elif version == 'V3':
44 opsdb = OpsimDatabaseV3(database, driver=driver, host=host, port=port,
45 longstrings=longstrings, verbose=verbose)
46 else:
47 warnings.warn('Could not identify opsim database version; just using Database class instead')
48 opsdb = Database(database, driver=driver, host=host, port=port,
49 longstrings=longstrings, verbose=verbose)
50 return opsdb
53class BaseOpsimDatabase(Database):
54 """Base opsim database class to gather common methods among different versions of the opsim schema.
56 Not intended to be used directly; use OpsimDatabaseFBS, OpsimDatabaseV3 or OpsimDatabaseV4 instead."""
57 def __init__(self, database, driver='sqlite', host=None, port=None, defaultTable=None,
58 longstrings=False, verbose=False):
59 super(BaseOpsimDatabase, self).__init__(database=database, driver=driver, host=host, port=port,
60 defaultTable=defaultTable, longstrings=longstrings,
61 verbose=verbose)
62 # Save filterlist so that we get the filter info per proposal in this desired order.
63 self.filterlist = np.array(['u', 'g', 'r', 'i', 'z', 'y'])
64 self.defaultTable = defaultTable
65 self._colNames()
67 def _colNames(self):
68 # Add version-specific column names in subclasses.
69 self.opsimVersion = 'unknown'
70 pass
72 def fetchMetricData(self, colnames, sqlconstraint=None, groupBy='default', tableName=None):
73 """
74 Fetch 'colnames' from 'tableName'.
76 Parameters
77 ----------
78 colnames : list
79 The columns to fetch from the table.
80 sqlconstraint : str, opt
81 The sql constraint to apply to the data (minus "WHERE"). Default None.
82 Examples: to fetch data for the r band filter only, set sqlconstraint to 'filter = "r"'.
83 groupBy : str, opt
84 The column to group the returned data by.
85 Default (when using summaryTable) is the MJD, otherwise will be None.
86 tableName : str, opt
87 The table to query. The default (None) will use the summary table, set by self.summaryTable.
89 Returns
90 -------
91 np.recarray
92 A structured array containing the data queried from the database.
93 """
94 if tableName is None:
95 tableName = self.defaultTable
96 if groupBy == 'default' and tableName==self.defaultTable:
97 groupBy = self.mjdCol
98 if groupBy == 'default' and tableName!=self.defaultTable:
99 groupBy = None
100 metricdata = super(BaseOpsimDatabase, self).fetchMetricData(colnames=colnames,
101 sqlconstraint=sqlconstraint,
102 groupBy=groupBy, tableName=tableName)
103 return metricdata
105 def fetchFieldsFromSummaryTable(self, sqlconstraint=None, raColName=None, decColName=None):
106 """
107 Fetch field information (fieldID/RA/Dec) from the summary table.
109 This implicitly only selects fields which were actually observed by opsim.
111 Parameters
112 ----------
113 sqlconstraint : str, opt
114 Sqlconstraint to apply before selecting observations to use for RA/Dec. Default None.
115 raColName : str, opt
116 Name of the RA column in the database.
117 decColName : str, opt
118 Name of the Dec column in the database.
119 degreesToRadians : bool, opt
120 Convert ra/dec into degrees?
121 If field information in summary table is in degrees, degreesToRadians should be True.
123 Returns
124 -------
125 np.recarray
126 Structured array containing the field data (fieldID, fieldRA, fieldDec). RA/Dec in radians.
127 """
128 if raColName is None:
129 raColName = self.raCol
130 if decColName is None:
131 decColName = self.decCol
132 fielddata = self.query_columns(self.defaultTable,
133 colnames=[self.fieldIdCol, raColName, decColName],
134 sqlconstraint=sqlconstraint, groupBy=self.fieldIdCol)
135 if self.raDecInDeg:
136 fielddata[raColName] = np.radians(fielddata[raColName])
137 fielddata[decColName] = np.radians(fielddata[decColName])
138 return fielddata
140 def fetchRunLength(self):
141 """Find the survey duration for a particular opsim run (years).
143 Returns
144 -------
145 float
146 """
147 if 'Config' not in self.tables:
148 print('Cannot access Config table to retrieve runLength; using default 10 years')
149 runLength = 10.0
150 else:
151 query = 'select paramValue from Config where paramName="%s"' % (self.runLengthParam)
152 runLength = self.query_arbitrary(query, dtype=[('paramValue', float)])
153 runLength = runLength['paramValue'][0] # Years
154 return runLength
156 def fetchLatLonHeight(self):
157 """Returns the latitude, longitude, and height of the telescope used by the config file.
158 """
159 if 'Config' not in self.tables:
160 print('Cannot access Config table to retrieve site parameters; using sims.utils.Site instead.')
161 site = Site(name='LSST')
162 lat = site.latitude_rad
163 lon = site.longitude_rad
164 height = site.height
165 else:
166 lat = self.query_columns('Config', colnames=['paramValue'],
167 sqlconstraint="paramName like '%latitude%'")
168 lat = float(lat['paramValue'][0])
169 lon = self.query_columns('Config', colnames=['paramValue'],
170 sqlconstraint="paramName like '%longitude%'")
171 lon = float(lon['paramValue'][0])
172 height = self.query_columns('Config', colnames=['paramValue'],
173 sqlconstraint="paramName like '%height%'")
174 height = float(height['paramValue'][0])
175 return lat, lon, height
177 def fetchOpsimRunName(self):
178 """Return opsim run name (machine name + session ID) from Session table.
179 """
180 if 'Session' not in self.tables:
181 print('Could not access Session table to find this information.')
182 runName = self.defaultRunName
183 else:
184 res = self.query_columns('Session', colnames=[self.sessionIdCol,
185 self.sessionHostCol])
186 runName = str(res[self.sessionHostCol][0]) + '_' + str(res[self.sessionIdCol][0])
187 return runName
189 def fetchNVisits(self, propId=None):
190 """Returns the total number of visits in the simulation or visits for a particular proposal.
192 Parameters
193 ----------
194 propId : int or list of ints
195 The ID numbers of the proposal(s).
197 Returns
198 -------
199 int
200 """
201 if 'ObsHistory' in self.tables and propId is None:
202 query = 'select count(*) from ObsHistory'
203 data = self.execute_arbitrary(query, dtype=([('nvisits', int)]))
204 else:
205 query = 'select count(distinct(%s)) from %s' %(self.mjdCol, self.defaultTable)
206 if propId is not None:
207 query += ' where '
208 if isinstance(propId, list) or isinstance(propId, np.ndarray):
209 for pID in propId:
210 query += 'propID=%d or ' %(int(pID))
211 query = query[:-3]
212 else:
213 query += 'propID = %d' %(int(propId))
214 data = self.execute_arbitrary(query, dtype=([('nvisits', int)]))
215 return data['nvisits'][0]
217 def fetchTotalSlewN(self):
218 """Return the total number of slews.
219 """
220 if 'SlewActivities' not in self.tables:
221 print('Could not access SlewActivities table to find this information.')
222 nslew = -1
223 else:
224 query = 'select count(distinct(%s)) from SlewActivities where %s >0' % (self.slewId,
225 self.delayCol)
226 res = self.execute_arbitrary(query, dtype=([('slewN', int)]))
227 nslew = res['slewN'][0]
228 return nslew
231class OpsimDatabaseFBS(BaseOpsimDatabase):
232 """
233 Database to class to interact with FBS versions of the opsim outputs.
235 Parameters
236 ----------
237 database : str
238 Name of the database or sqlite filename.
239 driver : str, opt
240 Name of the dialect + driver for sqlalchemy. Default 'sqlite'.
241 host : str, opt
242 Name of the database host. Default None (appropriate for sqlite files).
243 port : str, opt
244 String port number for the database. Default None (appropriate for sqlite files).
245 dbTables : dict, opt
246 Dictionary of the names of the tables in the database.
247 The dict should be key = table name, value = [table name, primary key].
248 """
249 def __init__(self, database, driver='sqlite', host=None, port=None, defaultTable='SummaryAllProps',
250 longstrings=False, verbose=False):
251 super().__init__(database=database, driver=driver, host=host, port=port,
252 defaultTable=defaultTable, longstrings=longstrings,
253 verbose=verbose)
255 def _colNames(self):
256 """
257 Set variables to represent the common column names used in this class directly.
259 This should make future schema changes a little easier to handle.
260 It is NOT meant to function as a general column map, just to abstract values
261 which are used *within this class*.
262 """
263 self.mjdCol = column('observationStartMJD')
264 self.raCol = column('fieldRA')
265 self.decCol = column('fieldDec')
266 self.propIdCol = column('proposalId')
267 # For config parsing.
268 self.versionCol = 'featureScheduler version'
269 self.dateCol = 'Date, ymd'
270 self.raDecInDeg = True
271 self.opsimVersion = 'FBS'
273 def fetchPropInfo(self):
274 """
275 There is no inherent proposal information or mapping in the FBS output.
276 An afterburner script does identify which visits may be counted as contributing toward WFD
277 and identifies these as such in the proposalID column (0 = general, 1 = WFD, 2+ = DD).
278 Returns dictionary of propID / propname, and dictionary of propTag / propID.
279 """
280 if 'Proposal' not in self.tables:
281 print('No proposal table available - no proposalIds have been assigned.')
282 return {}, {}
284 propIds = {}
285 # Add WFD and DD tags by default to propTags as we expect these every time. (avoids key errors).
286 propTags = {'WFD': [], 'DD': [], 'NES': []}
287 propData = self.query_columns('Proposal', colnames =[self.propIdCol, 'proposalName', 'proposalType'],
288 sqlconstraint=None)
289 for propId, propName, propType in zip(propData[self.propIdCol],
290 propData['proposalName'],
291 propData['proposalType']):
292 propIds[propId] = propName
293 if propType == 'WFD':
294 propTags['WFD'].append(propId)
295 if propType == 'DD':
296 propTags['DD'].append(propId)
297 return propIds, propTags
299 def createSQLWhere(self, tag, propTags):
300 """
301 Create a SQL constraint to identify observations taken for a particular proposal,
302 using the information in the propTags dictionary.
304 Parameters
305 ----------
306 tag : str
307 The name of the proposal for which to create a SQLwhere clause (WFD or DD).
308 propTags : dict
309 A dictionary of {proposal name : [proposal ids]}
310 This can be created using OpsimDatabase.fetchPropInfo()
312 Returns
313 -------
314 str
315 The SQL constraint, such as '(proposalID = 1) or (proposalID = 2)'
316 """
317 if (tag not in propTags) or (len(propTags[tag]) == 0):
318 print('No %s proposals found' % (tag))
319 # Create a sqlWhere clause that will not return anything as a query result.
320 sqlWhere = 'proposalId like "NO PROP"'
321 elif len(propTags[tag]) == 1:
322 sqlWhere = "proposalId = %d" % (propTags[tag][0])
323 else:
324 sqlWhere = "(" + " or ".join(["proposalId = %d" % (propid) for propid in propTags[tag]]) + ")"
325 return sqlWhere
327 def fetchConfig(self):
328 """
329 Fetch config data from configTable, match proposal IDs with proposal names and some field data,
330 and do a little manipulation of the data to make it easier to add to the presentation layer.
331 """
332 # Check to see if we're dealing with a full database or not. If not, just return (no config info to fetch).
333 if 'info' not in self.tables:
334 warnings.warn('Cannot fetch FBS config info.')
335 return {}, {}
336 # Create two dictionaries: a summary dict that contains a summary of the run
337 configSummary = {}
338 configSummary['keyorder'] = ['Version', 'RunInfo' ]
339 # and the other a general dict that contains all the details (by group) of the run.
340 config = {}
341 # Start to build up the summary.
342 # MAF version
343 mafdate, mafversion = getDateVersion()
344 configSummary['Version'] = {}
345 configSummary['Version']['MAFVersion'] = '%s' %(mafversion['__version__'])
346 configSummary['Version']['MAFDate'] = '%s' %(mafdate)
347 # Opsim date, version and runcomment info from config info table.
348 results = self.query_columns('info', ['Parameter', 'Value'],
349 sqlconstraint=f'Parameter like "{self.versionCol}"')
350 configSummary['Version']['OpsimVersion'] = 'FBS %s' %(results['Value'][0])
351 results = self.query_columns('info', ['Parameter', 'Value'],
352 sqlconstraint=f'Parameter like "{self.dateCol}"')
353 opsimdate = '-'.join(results['Value'][0].split(',')).replace(' ', '')
354 configSummary['Version']['OpsimDate'] = '%s' %(opsimdate)
355 configSummary['RunInfo'] = {}
356 results = self.query_columns('info', ['Parameter', 'Value'],
357 sqlconstraint=f'Parameter like "exec command%"')
358 configSummary['RunInfo']['Exec'] = '%s' % (results['Value'][0])
360 # Echo info table into configDetails.
361 configDetails = {}
362 configs = self.query_columns('info', ['Parameter', 'Value'])
363 for name, value in zip(configs['Parameter'], configs['Value']):
364 configDetails[name] = value
365 return configSummary, configDetails
368class OpsimDatabaseV4(BaseOpsimDatabase):
369 """
370 Database to class to interact with v4 versions of the opsim outputs.
372 Parameters
373 ----------
374 database : str
375 Name of the database or sqlite filename.
376 driver : str, opt
377 Name of the dialect + driver for sqlalchemy. Default 'sqlite'.
378 host : str, opt
379 Name of the database host. Default None (appropriate for sqlite files).
380 port : str, opt
381 String port number for the database. Default None (appropriate for sqlite files).
382 dbTables : dict, opt
383 Dictionary of the names of the tables in the database.
384 The dict should be key = table name, value = [table name, primary key].
385 """
386 def __init__(self, database, driver='sqlite', host=None, port=None, defaultTable='SummaryAllProps',
387 longstrings=False, verbose=False):
388 super(OpsimDatabaseV4, self).__init__(database=database, driver=driver, host=host, port=port,
389 defaultTable=defaultTable, longstrings=longstrings,
390 verbose=verbose)
392 def _colNames(self):
393 """
394 Set variables to represent the common column names used in this class directly.
396 This should make future schema changes a little easier to handle.
397 It is NOT meant to function as a general column map, just to abstract values
398 which are used *within this class*.
399 """
400 self.mjdCol = 'observationStartMJD'
401 self.slewId = 'slewHistory_slewCount'
402 self.delayCol = 'activityDelay'
403 self.fieldIdCol = 'fieldId'
404 self.raCol = 'fieldRA'
405 self.decCol = 'fieldDec'
406 self.propIdCol = 'propId'
407 self.propNameCol = 'propName'
408 self.propTypeCol = 'propType'
409 # For config parsing.
410 self.versionCol = 'version'
411 self.sessionIdCol = 'sessionId'
412 self.sessionHostCol = 'sessionHost'
413 self.sessionDateCol = 'sessionDate'
414 self.runCommentCol = 'runComment'
415 self.runLengthParam = 'survey/duration'
416 self.raDecInDeg = True
417 self.opsimVersion = 'V4'
419 def fetchFieldsFromFieldTable(self, propId=None, degreesToRadians=True):
420 """
421 Fetch field information (fieldID/RA/Dec) from the Field table.
423 This will select fields which were requested by a particular proposal or proposals,
424 even if they did not receive any observations.
426 Parameters
427 ----------
428 propId : int or list of ints
429 Proposal ID or list of proposal IDs to use to select fields.
430 degreesToRadians : bool, opt
431 If True, convert degrees in Field table into radians.
433 Returns
434 -------
435 np.recarray
436 Structured array containing the field data (fieldID, fieldRA, fieldDec).
437 """
438 if propId is not None:
439 query = 'select f.fieldId, f.ra, f.dec from Field as f'
440 query += ', ProposalField as p where (p.Field_fieldId = f.fieldId) '
441 if isinstance(propId, list) or isinstance(propId, np.ndarray):
442 query += ' and ('
443 for pID in propId:
444 query += '(p.Proposal_propId = %d) or ' % (int(pID))
445 # Remove the trailing 'or' and add a closing parenthesis.
446 query = query[:-3]
447 query += ')'
448 else: # single proposal ID.
449 query += ' and (p.Proposal_propId = %d) ' %(int(propId))
450 query += ' group by f.%s' %(self.fieldIdCol)
451 fielddata = self.query_arbitrary(query, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol],
452 ['int', 'float', 'float'])))
453 if len(fielddata) == 0:
454 fielddata = np.zeros(0, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol],
455 ['int', 'float', 'float'])))
456 else:
457 query = 'select fieldId, ra, dec from Field group by fieldId'
458 fielddata = self.query_arbitrary(query, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol],
459 ['int', 'float', 'float'])))
460 if len(fielddata) == 0:
461 fielddata = np.zeros(0, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol],
462 ['int', 'float', 'float'])))
463 if degreesToRadians:
464 fielddata[self.raCol] = fielddata[self.raCol] * np.pi / 180.
465 fielddata[self.decCol] = fielddata[self.decCol] * np.pi / 180.
466 return fielddata
469 def fetchPropInfo(self):
470 """
471 Fetch the proposal IDs as well as their (short) proposal names and science type tags from the
472 full opsim database.
473 Returns dictionary of propID / propname, and dictionary of propTag / propID.
474 If not using a full database, will return dict of propIDs with empty propnames + empty propTag dict.
475 """
476 propIds = {}
477 # Add WFD and DD tags by default to propTags as we expect these every time. (avoids key errors).
478 propTags = {'WFD': [], 'DD': [], 'NES': []}
479 assignData = True
480 try:
481 propData = self.query_columns('Proposal', colnames=[self.propIdCol, self.propNameCol],
482 sqlconstraint=None)
483 except ValueError:
484 propData = []
485 propIds = {}
486 propTags = {}
487 assignData = False
489 if assignData:
490 for propId, propName in zip(propData[self.propIdCol], propData[self.propNameCol]):
491 # Fix these in the future, to use the proper tags that will be added to output database.
492 propIds[propId] = propName
493 if 'widefastdeep' in propName.lower():
494 propTags['WFD'].append(propId)
495 if 'drilling' in propName.lower():
496 propTags['DD'].append(propId)
497 if 'northeclipticspur' in propName.lower():
498 propTags['NES'].append(propId)
499 return propIds, propTags
501 def createSlewConstraint(self, startTime=None, endTime=None):
502 """Create a SQL constraint for the slew tables (slew activities, slew speeds, slew states)
503 to select slews between startTime and endTime (MJD).
505 Parameters
506 ----------
507 startTime : float or None, opt
508 Time of first slew. Default (None) means no constraint on time of first slew.
509 endTime : float or None, opt
510 Time of last slew. Default (None) means no constraint on time of last slew.
512 Returns
513 -------
514 str
515 The SQL constraint, like 'slewHistory_slewID between XXX and XXXX'
516 """
517 query = 'select min(observationStartMJD), max(observationStartMJD) from obsHistory'
518 res = self.query_arbitrary(query, dtype=[('mjdMin', 'int'), ('mjdMax', 'int')])
519 # If asking for constraints that are out of bounds of the survey, return None.
520 if startTime is None or (startTime < res['mjdMin'][0]):
521 startTime = res['mjdMin'][0]
522 if endTime is None or (endTime > res['mjdMax'][0]):
523 endTime = res['mjdMax'][0]
524 # Check if times were out of bounds.
525 if startTime > res['mjdMax'][0] or endTime < res['mjdMin'][0]:
526 warnings.warn('Times requested do not overlap survey operations (%f to %f)'
527 % (res['mjdMin'][0], res['mjdMax'][0]))
528 return None
529 # Find the slew ID matching the start Time.
530 query = 'select min(observationId) from obsHistory where observationStartMJD >= %s' % (startTime)
531 res = self.query_arbitrary(query, dtype=[('observationId', 'int')])
532 obsHistId = res['observationId'][0]
533 query = 'select slewCount from slewHistory where ObsHistory_observationId = %d' % (obsHistId)
534 res = self.query_arbitrary(query, dtype=[('slewCount', 'int')])
535 minSlewCount = res['slewCount'][0]
536 # Find the slew ID matching the end Time.
537 query = 'select max(observationId) from obsHistory where observationStartMJD <= %s' % (endTime)
538 res = self.query_arbitrary(query, dtype=[('observationId', 'int')])
539 obsHistId = res['observationId'][0]
540 # Find the observation id corresponding to this slew time.
541 query = 'select slewCount from slewHistory where ObsHistory_observationId = %d' % (obsHistId)
542 res = self.query_arbitrary(query, dtype=[('slewCount', 'int')])
543 maxSlewCount = res['slewCount'][0]
544 return 'SlewHistory_slewCount between %d and %d' % (minSlewCount, maxSlewCount)
546 def createSQLWhere(self, tag, propTags):
547 """
548 Create a SQL constraint to identify observations taken for a particular proposal,
549 using the information in the propTags dictionary.
551 Parameters
552 ----------
553 tag : str
554 The name of the proposal for which to create a SQLwhere clause (WFD or DD).
555 propTags : dict
556 A dictionary of {proposal name : [proposal ids]}
557 This can be created using OpsimDatabase.fetchPropInfo()
559 Returns
560 -------
561 str
562 The SQL constraint, such as '(propID = 365) or (propID = 366)'
563 """
564 if (tag not in propTags) or (len(propTags[tag]) == 0):
565 print('No %s proposals found' % (tag))
566 # Create a sqlWhere clause that will not return anything as a query result.
567 sqlWhere = 'proposalId like "NO PROP"'
568 elif len(propTags[tag]) == 1:
569 sqlWhere = "proposalId = %d" % (propTags[tag][0])
570 else:
571 sqlWhere = "(" + " or ".join(["proposalId = %d" % (propid) for propid in propTags[tag]]) + ")"
572 return sqlWhere
574 def fetchRequestedNvisits(self, propId=None):
575 """Find the requested number of visits for the simulation or proposal(s).
577 Parameters
578 ----------
579 propId : int or list of ints, opt
581 Returns
582 -------
583 dict
584 Number of visits in u/g/r/i/z/y.
585 """
586 if propId is None:
587 constraint = ''
588 else:
589 if isinstance(propId, int):
590 constraint = 'propId = %d' % (propId)
591 else:
592 constraint = ''
593 for pId in propId:
594 constraint += 'propId = %d or ' % (int(pId))
595 constraint = constraint[:-3]
596 propData = self.query_columns('Proposal', colnames=[self.propNameCol, self.propTypeCol],
597 sqlconstraint=constraint)
598 nvisits = {}
599 for f in self.filterlist:
600 nvisits[f] = 0
601 for pName, propType in zip(propData[self.propNameCol], propData[self.propTypeCol]):
602 if propType.lower() == 'general':
603 for f in self.filterlist:
604 constraint = 'paramName="science/general_props/values/%s/filters/%s/num_visits"' \
605 % (pName, f)
606 val = self.query_columns('Config', colnames=['paramValue'], sqlconstraint=constraint)
607 if len(val) > 0:
608 nvisits[f] += int(val['paramValue'][0])
609 elif propType.lower == 'sequence':
610 pass
611 # Not clear yet.
612 return nvisits
614 def _matchParamNameValue(self, configarray, keyword):
615 return configarray['paramValue'][np.where(configarray['paramName'] == keyword)]
617 def _parseSequences(self, perPropConfig, filterlist):
618 """
619 (Private). Given an array of config paramName/paramValue info for a given WLTSS proposal
620 and the filterlist of filters used in that proposal, parse the sequences/subsequences and returns:
621 a dictionary with all the per-sequence information (including subsequence names & events, etc.)
622 a numpy array with the number of visits per filter
623 """
624 propDict = {}
625 # Identify where subsequences start in config[propname] arrays.
626 seqidxs = np.where(perPropConfig['paramName'] == 'SubSeqName')[0]
627 for sidx in seqidxs:
628 i = sidx
629 # Get the name of this subsequence.
630 seqname = perPropConfig['paramValue'][i]
631 # Check if seqname is a nested subseq of an existing sequence:
632 nestedsubseq = False
633 for prevseq in propDict:
634 if 'SubSeqNested' in propDict[prevseq]:
635 if seqname in propDict[prevseq]['SubSeqNested']:
636 seqdict = propDict[prevseq]['SubSeqNested'][seqname]
637 nestedsubseq = True
638 # If not, then create a new subseqence key/dictionary for this subseq.
639 if not nestedsubseq:
640 propDict[seqname] = {}
641 seqdict = propDict[seqname]
642 # And move on to next parameters within subsequence set.
643 i += 1
644 if perPropConfig['paramName'][i] == 'SubSeqNested':
645 subseqnestedname = perPropConfig['paramValue'][i]
646 if subseqnestedname != '.':
647 # Have nested subsequence, so keep track of that here
648 # but will fill in info later.
649 seqdict['SubSeqNested'] = {}
650 # Set up nested dictionary for nested subsequence.
651 seqdict['SubSeqNested'][subseqnestedname] = {}
652 i += 1
653 subseqfilters = perPropConfig['paramValue'][i]
654 if subseqfilters != '.':
655 seqdict['Filters'] = subseqfilters
656 i += 1
657 subseqexp = perPropConfig['paramValue'][i]
658 if subseqexp != '.':
659 seqdict['SubSeqExp'] = subseqexp
660 i+= 1
661 subseqevents = perPropConfig['paramValue'][i]
662 seqdict['Events'] = int(subseqevents)
663 i+=2
664 subseqinterval = perPropConfig['paramValue'][i]
665 subseqint = np.array([subseqinterval.split('*')], 'float').prod()
666 # In days ..
667 subseqint *= 1/24.0/60.0/60.0
668 if subseqint > 1:
669 seqdict['SubSeqInt'] = '%.2f days' %(subseqint)
670 else:
671 subseqint *= 24.0
672 if subseqint > 1:
673 seqdict['SubSeqInt'] = '%.2f hours' %(subseqint)
674 else:
675 subseqint *= 60.0
676 seqdict['SubSeqInt'] = '%.3f minutes' %(subseqint)
677 # End of assigning subsequence info - move on to counting number of visits.
678 nvisits = np.zeros(len(filterlist), int)
679 for subseq in propDict:
680 subevents = propDict[subseq]['Events']
681 # Count visits from direct subsequences.
682 if 'SubSeqExp' in propDict[subseq] and 'Filters' in propDict[subseq]:
683 subfilters = propDict[subseq]['Filters']
684 subexp = propDict[subseq]['SubSeqExp']
685 # If just one filter ..
686 if len(subfilters) == 1:
687 idx = np.where(filterlist == subfilters)[0]
688 nvisits[idx] += subevents * int(subexp)
689 else:
690 splitsubfilters = subfilters.split(',')
691 splitsubexp = subexp.split(',')
692 for f, exp in zip(splitsubfilters, splitsubexp):
693 idx = np.where(filterlist == f)[0]
694 nvisits[idx] += subevents * int(exp)
695 # Count visits if have nested subsequences.
696 if 'SubSeqNested' in propDict[subseq]:
697 for subseqnested in propDict[subseq]['SubSeqNested']:
698 events = subevents * propDict[subseq]['SubSeqNested'][subseqnested]['Events']
699 subfilters = propDict[subseq]['SubSeqNested'][subseqnested]['Filters']
700 subexp = propDict[subseq]['SubSeqNested'][subseqnested]['SubSeqExp']
701 # If just one filter ..
702 if len(subfilters) == 1:
703 idx = np.where(filterlist == subfilters)[0]
704 nvisits[idx] += events * int(subexp)
705 # Else may have multiple filters in the subsequence, so must split.
706 splitsubfilters = subfilters.split(',')
707 splitsubexp = subexp.split(',')
708 for f, exp in zip(splitsubfilters, splitsubexp):
709 idx = np.where(filterlist == f)[0]
710 nvisits[idx] += int(exp) * events
711 return propDict, nvisits
713 def _queryParam(self, constraint):
714 results = self.query_columns('Config', colnames=['paramValue'], sqlconstraint=constraint)
715 if len(results) > 0:
716 return results['paramValue'][0]
717 else:
718 return '--'
720 def fetchConfig(self):
721 """
722 Fetch config data from configTable, match proposal IDs with proposal names and some field data,
723 and do a little manipulation of the data to make it easier to add to the presentation layer.
724 """
725 # Check to see if we're dealing with a full database or not. If not, just return (no config info to fetch).
726 if 'Session' not in self.tables:
727 warnings.warn('Cannot fetch opsim config info as this is not a full opsim database.')
728 return {}, {}
729 # Create two dictionaries: a summary dict that contains a summary of the run
730 configSummary = {}
731 configSummary['keyorder'] = ['Version', 'RunInfo', 'Proposals']
732 # and the other a general dict that contains all the details (by group) of the run.
733 config = {}
734 # Start to build up the summary.
735 # MAF version
736 mafdate, mafversion = getDateVersion()
737 configSummary['Version'] = {}
738 configSummary['Version']['MAFVersion'] = '%s' %(mafversion['__version__'])
739 configSummary['Version']['MAFDate'] = '%s' %(mafdate)
740 # Opsim date, version and runcomment info from session table
741 results = self.query_columns('Session',
742 [self.versionCol, self.sessionDateCol, self.runCommentCol])
743 configSummary['Version']['OpsimVersion'] = '%s' %(results[self.versionCol][0])
744 configSummary['Version']['OpsimDate'] = '%s' %(results[self.sessionDateCol][0])
745 configSummary['Version']['OpsimDate'] = configSummary['Version']['OpsimDate'][0:10]
746 configSummary['RunInfo'] = {}
747 configSummary['RunInfo']['RunComment'] = results[self.runCommentCol][0]
748 configSummary['RunInfo']['RunName'] = self.fetchOpsimRunName()
749 # Pull out a few special values to put into summary.
750 # This section has a number of configuration parameter names hard-coded.
751 # I've left these here (rather than adding to self_colNames), bc I think schema changes in the config
752 # files will actually be easier to track here (at least until the opsim configs are cleaned up).
753 constraint = 'paramName="observatory/telescope/altitude_minpos"'
754 configSummary['RunInfo']['MinAlt'] = self._queryParam(constraint)
755 constraint = 'paramName="observatory/telescope/altitude_maxpos"'
756 configSummary['RunInfo']['MaxAlt'] = self._queryParam(constraint)
757 constraint = 'paramName="observatory/camera/filter_change_time"'
758 configSummary['RunInfo']['TimeFilterChange'] = self._queryParam(constraint)
759 constraint = 'paramName="observatory/camera/readout_time"'
760 configSummary['RunInfo']['TimeReadout'] = self._queryParam(constraint)
761 constraint = 'paramName="sched_driver/propboost_weight"'
762 configSummary['RunInfo']['PropBoostWeight'] = self._queryParam(constraint)
763 configSummary['RunInfo']['keyorder'] = ['RunName', 'RunComment', 'MinAlt', 'MaxAlt',
764 'TimeFilterChange', 'TimeReadout', 'PropBoostWeight']
766 # Echo config table into configDetails.
767 configDetails = {}
768 configs = self.query_columns('Config', ['paramName', 'paramValue'])
769 for name, value in zip(configs['paramName'], configs['paramValue']):
770 configDetails[name] = value
772 # Now finish building the summary to add proposal information.
773 # Loop through all proposals to add summary information.
774 propData = self.query_columns('Proposal', [self.propIdCol, self.propNameCol, self.propTypeCol])
775 configSummary['Proposals'] = {}
776 for propid, propname, proptype in zip(propData[self.propIdCol],
777 propData[self.propNameCol], propData[self.propTypeCol]):
778 configSummary['Proposals'][propname] = {}
779 propdict = configSummary['Proposals'][propname]
780 propdict['keyorder'] = ['PropId', 'PropName', 'PropType', 'Airmass bonus', 'Airmass max',
781 'HA bonus', 'HA max', 'Time weight', 'Restart Lost Sequences',
782 'Restart Complete Sequences', 'Filters']
783 propdict['PropName'] = propname
784 propdict['PropId'] = propid
785 propdict['PropType'] = proptype
786 # Add some useful information on the proposal parameters.
787 constraint = 'paramName like "science/%s_props/values/%s/sky_constraints/max_airmass"'\
788 % ("%", propname)
789 propdict['Airmass max'] = self._queryParam(constraint)
790 constraint = 'paramName like "science/%s_props/values/%s/scheduling/airmass_bonus"'\
791 % ("%", propname)
792 propdict['Airmass bonus'] = self._queryParam(constraint)
793 constraint = 'paramName like "science/%s_props/values/%s/scheduling/hour_angle_max"'\
794 % ("%", propname)
795 propdict['HA max'] = self._queryParam(constraint)
796 constraint = 'paramName like "science/%s_props/values/%s/scheduling/hour_angle_bonus"'\
797 % ("%", propname)
798 propdict['HA bonus'] = self._queryParam(constraint)
799 constraint = 'paramName like "science/%s_props/values/%s/scheduling/time_weight"'\
800 % ("%", propname)
801 propdict['Time weight'] = self._queryParam(constraint)
802 constraint = 'paramName like "science/%s_props/values/%s/scheduling/restart_lost_sequences"'\
803 % ("%", propname)
804 propdict['Restart Lost Sequences'] = self._queryParam(constraint)
805 constraint = 'paramName like "science/%s_props/values/%s/scheduling/restart_complete_sequences"'\
806 % ("%", propname)
807 propdict['Restart Complete Sequences'] = self._queryParam(constraint)
808 # Find number of visits requested per filter for the proposal
809 # along with min/max sky and airmass values.
810 propdict['Filters'] = {}
811 for f in self.filterlist:
812 propdict['Filters'][f] = {}
813 propdict['Filters'][f]['Filter'] = f
814 dictkeys = ['MaxSeeing', 'BrightLimit', 'DarkLimit', 'NumVisits', 'GroupedVisits', 'Snaps']
815 querykeys = ['max_seeing', 'bright_limit', 'dark_limit', 'num_visits',
816 'num_grouped_visits', 'exposures']
817 for dk, qk in zip(dictkeys, querykeys):
818 constraint = 'paramName like "science/%s_props/values/%s/filters/%s/%s"' \
819 % ("%", propname, f, qk)
820 propdict['Filters'][f][dk] = self._queryParam(constraint)
821 propdict['Filters'][f]['keyorder'] = ['Filter', 'MaxSeeing', 'MinSky', 'MaxSky',
822 'NumVisits', 'GroupedVisits', 'Snaps']
823 propdict['Filters']['keyorder'] = list(self.filterlist)
824 return configSummary, configDetails
827class OpsimDatabaseV3(BaseOpsimDatabase):
828 def __init__(self, database, driver='sqlite', host=None, port=None, defaultTable='Summary',
829 longstrings=False, verbose=False):
830 """
831 Instantiate object to handle queries of the opsim database.
832 (In general these will be the sqlite database files produced by opsim, but could
833 be any database holding those opsim output tables.).
835 database = Name of database or sqlite filename
836 driver = Name of database dialect+driver for sqlalchemy (e.g. 'sqlite', 'pymssql+mssql')
837 host = Name of database host (optional)
838 port = String port number (optional)
840 """
841 super(OpsimDatabaseV3, self).__init__(database=database, driver=driver, host=host, port=port,
842 defaultTable=defaultTable, longstrings=longstrings,
843 verbose=verbose)
845 def _colNames(self):
846 """
847 Set variables to represent the common column names used in this class directly.
849 This should make future schema changes a little easier to handle.
850 It is NOT meant to function as a general column map, just to abstract values
851 which are used *within this class*.
852 """
853 self.mjdCol = 'expMJD'
854 self.slewId = 'slewHistory_slewID'
855 self.delayCol = 'actDelay'
856 self.fieldIdCol = 'fieldID'
857 self.raCol = 'fieldRA'
858 self.decCol = 'fieldDec'
859 self.propIdCol = 'propID'
860 self.propConfCol = 'propConf'
861 self.propNameCol = 'propName' #(propname == proptype)
862 # For config parsing.
863 self.versionCol = 'version'
864 self.sessionIdCol = 'sessionID'
865 self.sessionHostCol = 'sessionHost'
866 self.sessionDateCol = 'sessionDate'
867 self.runCommentCol = 'runComment'
868 self.runLengthParam = 'nRun'
869 self.raDecInDeg = False
870 self.opsimVersion = 'V3'
872 def fetchFieldsFromFieldTable(self, propId=None, degreesToRadians=True):
873 """
874 Fetch field information (fieldID/RA/Dec) from Field (+Proposal_Field) tables.
876 propID = the proposal ID (default None), if selecting particular proposal - can be a list
877 degreesToRadians = RA/Dec values are in degrees in the Field table (so convert to radians).
878 """
879 # Note that you can't select any other sql constraints (such as filter).
880 # This will select fields which were requested by a particular proposal or proposals,
881 # even if they didn't get any observations.
882 if propId is not None:
883 query = 'select f.%s, f.%s, f.%s from %s as f' %(self.fieldIdCol, self.raCol, self.decCol,
884 'Field')
885 query += ', %s as p where (p.Field_%s = f.%s) ' %('Proposal_Field',
886 self.fieldIdCol, self.fieldIdCol)
887 if isinstance(propId, list) or isinstance(propId, np.ndarray):
888 query += ' and ('
889 for pID in propId:
890 query += '(p.Proposal_%s = %d) or ' %(self.propIdCol, int(pID))
891 # Remove the trailing 'or' and add a closing parenthesis.
892 query = query[:-3]
893 query += ')'
894 else: # single proposal ID.
895 query += ' and (p.Proposal_%s = %d) ' %(self.propIdCol, int(propId))
896 query += ' group by f.%s' %(self.fieldIdCol)
897 fielddata = self.query_arbitrary(query, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol],
898 ['int', 'float', 'float'])))
899 if len(fielddata) == 0:
900 fielddata = np.zeros(0, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol],
901 ['int', 'float', 'float'])))
902 else:
903 fielddata = self.query_columns('Field', colnames=[self.fieldIdCol, self.raCol, self.decCol],
904 groupBy = self.fieldIdCol)
905 if degreesToRadians:
906 fielddata[self.raCol] = fielddata[self.raCol] * np.pi / 180.
907 fielddata[self.decCol] = fielddata[self.decCol] * np.pi / 180.
908 return fielddata
910 def fetchPropInfo(self):
911 """
912 Fetch the proposal IDs as well as their (short) proposal names and science type tags from the
913 full opsim database.
914 Returns dictionary of propID / propname, and dictionary of propTag / propID.
915 If not using a full database, will return dict of propIDs with empty propnames + empty propTag dict.
916 """
917 propIDs = {}
918 # Add WFD and DD tags by default to propTags as we expect these every time. (avoids key errors).
919 propTags = {'WFD':[], 'DD':[], 'NES': []}
920 # If do not have full database available:
921 if 'Proposal' not in self.tables:
922 propData = self.query_columns(self.defaultTable, colnames=[self.propIdCol])
923 for propid in propData[self.propIdCol]:
924 propIDs[int(propid)] = propid
925 else:
926 # Query for all propIDs.
927 propData = self.query_columns('Proposal', colnames=[self.propIdCol, self.propConfCol,
928 self.propNameCol], sqlconstraint=None)
929 for propid, propname in zip(propData[self.propIdCol], propData[self.propConfCol]):
930 # Strip '.conf', 'Prop', and path info.
931 propIDs[int(propid)] = re.sub('Prop','', re.sub('.conf','', re.sub('.*/', '', propname)))
932 # Find the 'ScienceType' from the config table, to indicate DD/WFD/Rolling, etc.
933 sciencetypes = self.query_columns('Config', colnames=['paramValue', 'nonPropID'],
934 sqlconstraint="paramName like 'ScienceType'")
935 if len(sciencetypes) == 0:
936 # Then this was an older opsim run without 'ScienceType' tags,
937 # so fall back to trying to guess what proposals are WFD or DD.
938 for propid, propname in propIDs.items():
939 if 'universal' in propname.lower():
940 propTags['WFD'].append(propid)
941 if 'deep' in propname.lower():
942 propTags['DD'].append(propid)
943 if 'northecliptic' in propname.lower():
944 propTags['NES'].append(propid)
945 else:
946 # Newer opsim output with 'ScienceType' fields in conf files.
947 for sc in sciencetypes:
948 # ScienceType tag can be multiple values, separated by a ','
949 tags = [x.strip(' ') for x in sc['paramValue'].split(',')]
950 for sciencetype in tags:
951 if sciencetype in propTags:
952 propTags[sciencetype].append(int(sc['nonPropID']))
953 else:
954 propTags[sciencetype] = [int(sc['nonPropID']),]
955 # But even these runs don't tag NES
956 for propid, propname in propIDs.items():
957 if 'northecliptic' in propname.lower():
958 propTags['NES'].append(propid)
959 return propIDs, propTags
961 def createSlewConstraint(self, startTime=None, endTime=None):
962 """Create a SQL constraint for the slew tables (slew activities, slew speeds, slew states)
963 to select slews between startTime and endTime (MJD).
965 Parameters
966 ----------
967 startTime : float or None, opt
968 Time of first slew. Default (None) means no constraint on time of first slew.
969 endTime : float or None, opt
970 Time of last slew. Default (None) means no constraint on time of last slew.
972 Returns
973 -------
974 str
975 The SQL constraint, like 'slewHistory_slewID > XXX and slewHistory_slewID < XXXX'
976 """
977 query = 'select min(expMJD), max(expMJD) from obsHistory'
978 res = self.query_arbitrary(query, dtype=[('mjdMin', 'int'), ('mjdMax', 'int')])
979 # If asking for constraints that are out of bounds of the survey, return None.
980 if startTime is None or (startTime < res['mjdMin'][0]):
981 startTime = res['mjdMin'][0]
982 if endTime is None or (endTime > res['mjdMax'][0]):
983 endTime = res['mjdMax'][0]
984 # Check if times were out of bounds.
985 if startTime > res['mjdMax'][0] or endTime < res['mjdMin'][0]:
986 warnings.warn('Times requested do not overlap survey operations (%f to %f)'
987 % (res['mjdMin'][0], res['mjdMax'][0]))
988 return None
989 # Find the slew ID matching the start Time.
990 query = 'select min(obsHistID) from obsHistory where expMJD >= %s' % (startTime)
991 res = self.query_arbitrary(query, dtype=[('observationId', 'int')])
992 obsHistId = res['observationId'][0]
993 query = 'select slewID from slewHistory where ObsHistory_obsHistID = %d' % (obsHistId)
994 res = self.query_arbitrary(query, dtype=[('slewCount', 'int')])
995 minSlewCount = res['slewCount'][0]
996 # Find the slew ID matching the end Time.
997 query = 'select max(obsHistID) from obsHistory where expMJD <= %s' % (endTime)
998 res = self.query_arbitrary(query, dtype=[('observationId', 'int')])
999 obsHistId = res['observationId'][0]
1000 # Find the observation id corresponding to this slew time.
1001 query = 'select slewID from slewHistory where ObsHistory_obsHistID = %d' % (obsHistId)
1002 res = self.query_arbitrary(query, dtype=[('slewCount', 'int')])
1003 maxSlewCount = res['slewCount'][0]
1004 return 'SlewHistory_slewID between %d and %d' % (minSlewCount, maxSlewCount)
1006 def createSQLWhere(self, tag, propTags):
1007 """Create a SQL constraint to identify observations taken for a particular proposal,
1008 using the information in the propTags dictionary.
1010 Parameters
1011 ----------
1012 tag : str
1013 The name of the proposal for which to create a SQLwhere clause (WFD or DD).
1014 propTags : dict
1015 A dictionary of {proposal name : [proposal ids]}
1016 This can be created using OpsimDatabase.fetchPropInfo()
1018 Returns
1019 -------
1020 str
1021 The SQL constraint, such as '(propID = 365) or (propID = 366)'
1022 """
1023 if (tag not in propTags) or (len(propTags[tag]) == 0):
1024 print('No %s proposals found' % (tag))
1025 # Create a sqlWhere clause that will not return anything as a query result.
1026 sqlWhere = 'propID like "NO PROP"'
1027 elif len(propTags[tag]) == 1:
1028 sqlWhere = "propID = %d" % (propTags[tag][0])
1029 else:
1030 sqlWhere = "(" + " or ".join(["propID = %d" % (propid) for propid in propTags[tag]]) + ")"
1031 return sqlWhere
1033 def fetchRequestedNvisits(self, propId=None):
1034 """
1035 Find the requested number of visits for proposals in propId.
1036 Returns a dictionary - Nvisits{u/g/r/i/z/y}
1037 """
1038 visitDict = {}
1039 if propId is None:
1040 # Get all the available propIds.
1041 propData = self.query_columns('Proposal', colnames=[self.propIdCol, self.propNameCol],
1042 sqlconstraint=None)
1043 else:
1044 # Get the propType info to go with the propId(s).
1045 if hasattr(propId, '__iter__'):
1046 constraint = '('
1047 for pi in propId:
1048 constraint += '(propId = %d) or ' %(pi)
1049 constraint = constraint[:-4] + ')'
1050 else:
1051 constraint = 'propId = %d' %(propId)
1052 propData = self.query_columns('Proposal', colnames=[self.propIdCol, self.propNameCol],
1053 sqlconstraint=constraint)
1054 for pId, propType in zip(propData[self.propIdCol], propData[self.propNameCol]):
1055 perPropConfig = self.query_columns('Config', colnames=['paramName', 'paramValue'],
1056 sqlconstraint = 'nonPropID = %d and paramName!="userRegion"'
1057 %(pId))
1058 filterlist = self._matchParamNameValue(perPropConfig, 'Filter')
1059 if propType == 'WL':
1060 # For WL proposals, the simple 'Filter_Visits' == the requested number of observations.
1061 nvisits = np.array(self._matchParamNameValue(perPropConfig, 'Filter_Visits'), int)
1062 elif propType == 'WLTSS':
1063 seqDict, nvisits = self._parseSequences(perPropConfig, filterlist)
1064 visitDict[pId] = {}
1065 for f, N in zip(filterlist, nvisits):
1066 visitDict[pId][f] = N
1067 nvisits = {}
1068 for f in ['u', 'g', 'r', 'i', 'z', 'y']:
1069 nvisits[f] = 0
1070 for pId in visitDict:
1071 for f in visitDict[pId]:
1072 nvisits[f] += visitDict[pId][f]
1073 return nvisits
1075 def _matchParamNameValue(self, configarray, keyword):
1076 return configarray['paramValue'][np.where(configarray['paramName']==keyword)]
1078 def _parseSequences(self, perPropConfig, filterlist):
1079 """
1080 (Private). Given an array of config paramName/paramValue info for a given WLTSS proposal
1081 and the filterlist of filters used in that proposal, parse the sequences/subsequences and returns:
1082 a dictionary with all the per-sequence information (including subsequence names & events, etc.)
1083 a numpy array with the number of visits per filter
1084 """
1085 propDict = {}
1086 # Identify where subsequences start in config[propname] arrays.
1087 seqidxs = np.where(perPropConfig['paramName'] == 'SubSeqName')[0]
1088 for sidx in seqidxs:
1089 i = sidx
1090 # Get the name of this subsequence.
1091 seqname = perPropConfig['paramValue'][i]
1092 # Check if seqname is a nested subseq of an existing sequence:
1093 nestedsubseq = False
1094 for prevseq in propDict:
1095 if 'SubSeqNested' in propDict[prevseq]:
1096 if seqname in propDict[prevseq]['SubSeqNested']:
1097 seqdict = propDict[prevseq]['SubSeqNested'][seqname]
1098 nestedsubseq = True
1099 # If not, then create a new subseqence key/dictionary for this subseq.
1100 if not nestedsubseq:
1101 propDict[seqname] = {}
1102 seqdict = propDict[seqname]
1103 # And move on to next parameters within subsequence set.
1104 i += 1
1105 if perPropConfig['paramName'][i] == 'SubSeqNested':
1106 subseqnestedname = perPropConfig['paramValue'][i]
1107 if subseqnestedname != '.':
1108 # Have nested subsequence, so keep track of that here
1109 # but will fill in info later.
1110 seqdict['SubSeqNested'] = {}
1111 # Set up nested dictionary for nested subsequence.
1112 seqdict['SubSeqNested'][subseqnestedname] = {}
1113 i += 1
1114 subseqfilters = perPropConfig['paramValue'][i]
1115 if subseqfilters != '.':
1116 seqdict['Filters'] = subseqfilters
1117 i += 1
1118 subseqexp = perPropConfig['paramValue'][i]
1119 if subseqexp != '.':
1120 seqdict['SubSeqExp'] = subseqexp
1121 i+= 1
1122 subseqevents = perPropConfig['paramValue'][i]
1123 seqdict['Events'] = int(subseqevents)
1124 i+=2
1125 subseqinterval = perPropConfig['paramValue'][i]
1126 subseqint = np.array([subseqinterval.split('*')], 'float').prod()
1127 # In days ..
1128 subseqint *= 1/24.0/60.0/60.0
1129 if subseqint > 1:
1130 seqdict['SubSeqInt'] = '%.2f days' %(subseqint)
1131 else:
1132 subseqint *= 24.0
1133 if subseqint > 1:
1134 seqdict['SubSeqInt'] = '%.2f hours' %(subseqint)
1135 else:
1136 subseqint *= 60.0
1137 seqdict['SubSeqInt'] = '%.3f minutes' %(subseqint)
1138 # End of assigning subsequence info - move on to counting number of visits.
1139 nvisits = np.zeros(len(filterlist), int)
1140 for subseq in propDict:
1141 subevents = propDict[subseq]['Events']
1142 # Count visits from direct subsequences.
1143 if 'SubSeqExp' in propDict[subseq] and 'Filters' in propDict[subseq]:
1144 subfilters = propDict[subseq]['Filters']
1145 subexp = propDict[subseq]['SubSeqExp']
1146 # If just one filter ..
1147 if len(subfilters) == 1:
1148 idx = np.where(filterlist == subfilters)[0]
1149 nvisits[idx] += subevents * int(subexp)
1150 else:
1151 splitsubfilters = subfilters.split(',')
1152 splitsubexp = subexp.split(',')
1153 for f, exp in zip(splitsubfilters, splitsubexp):
1154 idx = np.where(filterlist == f)[0]
1155 nvisits[idx] += subevents * int(exp)
1156 # Count visits if have nested subsequences.
1157 if 'SubSeqNested' in propDict[subseq]:
1158 for subseqnested in propDict[subseq]['SubSeqNested']:
1159 events = subevents * propDict[subseq]['SubSeqNested'][subseqnested]['Events']
1160 subfilters = propDict[subseq]['SubSeqNested'][subseqnested]['Filters']
1161 subexp = propDict[subseq]['SubSeqNested'][subseqnested]['SubSeqExp']
1162 # If just one filter ..
1163 if len(subfilters) == 1:
1164 idx = np.where(filterlist == subfilters)[0]
1165 nvisits[idx] += events * int(subexp)
1166 # Else may have multiple filters in the subsequence, so must split.
1167 splitsubfilters = subfilters.split(',')
1168 splitsubexp = subexp.split(',')
1169 for f, exp in zip(splitsubfilters, splitsubexp):
1170 idx = np.where(filterlist == f)[0]
1171 nvisits[idx] += int(exp) * events
1172 return propDict, nvisits
1174 def fetchConfig(self):
1175 """
1176 Fetch config data from configTable, match proposal IDs with proposal names and some field data,
1177 and do a little manipulation of the data to make it easier to add to the presentation layer.
1178 """
1179 # Check to see if we're dealing with a full database or not. If not, just return (no config info to fetch).
1180 if 'Session' not in self.tables:
1181 warnings.warn('Cannot fetch opsim config info as this is not a full opsim database.')
1182 return {}, {}
1183 # Create two dictionaries: a summary dict that contains a summary of the run
1184 configSummary = {}
1185 configSummary['keyorder'] = ['Version', 'RunInfo', 'Proposals']
1186 # and the other a general dict that contains all the details (by group) of the run.
1187 config = {}
1188 # Start to build up the summary.
1189 # MAF version
1190 mafdate, mafversion = getDateVersion()
1191 configSummary['Version'] = {}
1192 configSummary['Version']['MAFVersion'] = '%s' % (mafversion['__version__'])
1193 configSummary['Version']['MAFDate'] = '%s' % (mafdate)
1194 # Opsim date, version and runcomment info from session table
1195 results = self.query_columns('Session', colnames = [self.versionCol, self.sessionDateCol,
1196 self.runCommentCol])
1197 configSummary['Version']['OpsimVersion'] = '%s' % (results[self.versionCol][0])
1198 configSummary['Version']['OpsimDate'] = '%s' % (results[self.sessionDateCol][0])
1199 configSummary['RunInfo'] = {}
1200 configSummary['RunInfo']['RunComment'] = results[self.runCommentCol]
1201 configSummary['RunInfo']['RunName'] = self.fetchOpsimRunName()
1202 # Pull out a few special values to put into summary.
1203 # This section has a number of configuration parameter names hard-coded.
1204 # I've left these here (rather than adding to self_colNames), because I think schema changes will in the config
1205 # files will actually be easier to track here (at least until the opsim configs are cleaned up).
1206 constraint = 'moduleName="Config" and paramName="sha1"'
1207 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint)
1208 try:
1209 configSummary['Version']['Config sha1'] = results['paramValue'][0]
1210 except IndexError:
1211 configSummary['Version']['Config sha1'] = 'Unavailable'
1212 constraint = 'moduleName="Config" and paramName="changedFiles"'
1213 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint)
1214 try:
1215 configSummary['Version']['Config changed files'] = results['paramValue'][0]
1216 except IndexError:
1217 configSummary['Version']['Config changed files'] = 'Unavailable'
1218 constraint = 'moduleName="instrument" and paramName="Telescope_AltMin"'
1219 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint)
1220 configSummary['RunInfo']['MinAlt'] = results['paramValue'][0]
1221 constraint = 'moduleName="instrument" and paramName="Telescope_AltMax"'
1222 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint)
1223 configSummary['RunInfo']['MaxAlt'] = results['paramValue'][0]
1224 constraint = 'moduleName="instrument" and paramName="Filter_MoveTime"'
1225 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint)
1226 configSummary['RunInfo']['TimeFilterChange'] = results['paramValue'][0]
1227 constraint = 'moduleName="instrument" and paramName="Readout_Time"'
1228 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint)
1229 configSummary['RunInfo']['TimeReadout'] = results['paramValue'][0]
1230 constraint = 'moduleName="scheduler" and paramName="MinDistance2Moon"'
1231 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint)
1232 configSummary['RunInfo']['MinDist2Moon'] = results['paramValue'][0]
1233 configSummary['RunInfo']['keyorder'] = ['RunName', 'RunComment', 'MinDist2Moon', 'MinAlt', 'MaxAlt',
1234 'TimeFilterChange', 'TimeReadout']
1235 # Now build up config dict with 'nice' group names (proposal name and short module name)
1236 # Each dict entry is a numpy array with the paramName/paramValue/comment values.
1237 # Match proposal IDs with names.
1238 query = 'select %s, %s, %s from Proposal group by %s' %(self.propIdCol, self.propConfCol,
1239 self.propNameCol, self.propIdCol)
1240 propdata = self.query_arbitrary(query, dtype=([(self.propIdCol, int),
1241 (self.propConfCol, np.str_, 256),
1242 (self.propNameCol, np.str_, 256)]))
1243 # Make 'nice' proposal names
1244 propnames = np.array([os.path.split(x)[1].replace('.conf', '') for x in propdata[self.propConfCol]])
1245 # Get 'nice' module names
1246 moduledata = self.query_columns('Config', colnames=['moduleName',], sqlconstraint='nonPropID=0')
1247 modulenames = np.array([os.path.split(x)[1].replace('.conf', '') for x in moduledata['moduleName']])
1248 # Grab the config information for each proposal and module.
1249 cols = ['paramName', 'paramValue', 'comment']
1250 for longmodname, modname in zip(moduledata['moduleName'], modulenames):
1251 config[modname] = self.query_columns('Config', colnames=cols,
1252 sqlconstraint='moduleName="%s"' %(longmodname))
1253 config[modname] = config[modname][['paramName', 'paramValue', 'comment']]
1254 for propid, propname in zip(propdata[self.propIdCol], propnames):
1255 config[propname] = self.query_columns('Config', colnames=cols,
1256 sqlconstraint='nonPropID="%s" and paramName!="userRegion"'
1257 %(propid))
1258 config[propname] = config[propname][['paramName', 'paramValue', 'comment']]
1259 config['keyorder'] = ['Comment', 'LSST', 'site', 'instrument', 'filters',
1260 'AstronomicalSky', 'File', 'scheduler',
1261 'schedulingData', 'schedDown', 'unschedDown']
1262 # Now finish building the summary to add proposal information.
1263 # Loop through all proposals to add summary information.
1264 configSummary['Proposals'] = {}
1265 propidorder = sorted(propdata[self.propIdCol])
1266 # Generate a keyorder to print proposals in order of propid.
1267 configSummary['Proposals']['keyorder'] = []
1268 for propid in propidorder:
1269 configSummary['Proposals']['keyorder'].append(propnames[np.where(propdata[self.propIdCol]
1270 == propid)][0])
1271 for propid, propname in zip(propdata[self.propIdCol], propnames):
1272 configSummary['Proposals'][propname] = {}
1273 propdict = configSummary['Proposals'][propname]
1274 propdict['keyorder'] = [self.propIdCol, self.propNameCol, 'PropType', 'RelPriority',
1275 'NumUserRegions', 'NumFields']
1276 propdict[self.propNameCol] = propname
1277 propdict[self.propIdCol] = propid
1278 propdict['PropType'] = propdata[self.propNameCol][np.where(propnames == propname)]
1279 propdict['RelPriority'] = self._matchParamNameValue(config[propname], 'RelativeProposalPriority')
1280 # Get the number of user regions.
1281 constraint = 'nonPropID="%s" and paramName="userRegion"' %(propid)
1282 result = self.query_columns('Config', colnames=['paramName',], sqlconstraint=constraint)
1283 propdict['NumUserRegions'] = result.size
1284 # Get the number of fields requested in the proposal (all filters).
1285 propdict['NumFields'] = self.fetchFieldsFromFieldTable(propId=propid).size
1286 # Find number of visits requested per filter for the proposal, with min/max sky & airmass values.
1287 # Note that config table has multiple entries for Filter/Filter_Visits/etc. with the same name.
1288 # The order of these entries in the config array matters.
1289 propdict['PerFilter'] = {}
1290 for key, keyword in zip(['Filters', 'MaxSeeing', 'MinSky', 'MaxSky'],
1291 ['Filter', 'Filter_MaxSeeing', 'Filter_MinBrig', 'Filter_MaxBrig']):
1292 temp = self._matchParamNameValue(config[propname], keyword)
1293 if len(temp) > 0:
1294 propdict['PerFilter'][key] = temp
1295 # Add exposure time, potentially looking for scaling per filter.
1296 exptime = float(self._matchParamNameValue(config[propname], 'ExposureTime')[0])
1297 temp = self._matchParamNameValue(config[propname], 'Filter_ExpFactor')
1298 if len(temp) > 0:
1299 propdict['PerFilter']['VisitTime'] = temp * exptime
1300 else:
1301 propdict['PerFilter']['VisitTime'] = np.ones(len(propdict['PerFilter']['Filters']), float)
1302 propdict['PerFilter']['VisitTime'] *= exptime
1303 # And count how many total exposures are requested per filter.
1304 # First check if 'RestartCompleteSequences' is true:
1305 # if both are true, then basically an indefinite number of visits are requested, although we're
1306 # not going to count this way (as the proposals still make an approximate number of requests).
1307 restartComplete = False
1308 temp = self._matchParamNameValue(config[propname], 'RestartCompleteSequences')
1309 if len(temp) > 0:
1310 if temp[0] == 'True':
1311 restartComplete = True
1312 propdict['RestartCompleteSequences'] = restartComplete
1313 # Grab information on restarting lost sequences so we can print this too.
1314 restartLost = False
1315 tmp = self._matchParamNameValue(config[propname], 'RestartLostSequences')
1316 if len(temp) > 0:
1317 if temp[0] == 'True':
1318 restartLost = True
1319 propdict['RestartLostSequences'] = restartLost
1320 if propdict['PropType'] == 'WL':
1321 # Simple 'Filter_Visits' request for number of observations.
1322 propdict['PerFilter']['NumVisits'] = np.array(self._matchParamNameValue(config[propname],
1323 'Filter_Visits'), int)
1324 elif propdict['PropType'] == 'WLTSS':
1325 # Proposal contains subsequences and possible nested subseq, so must delve further.
1326 # Make a dictionary to hold the subsequence info (keyed per subsequence).
1327 propdict['SubSeq'], Nvisits = self._parseSequences(config[propname],
1328 propdict['PerFilter']['Filters'])
1329 propdict['PerFilter']['NumVisits'] = Nvisits
1330 propdict['SubSeq']['keyorder'] = ['SubSeqName', 'SubSeqNested', 'Events', 'SubSeqInt']
1331 # Sort the filter information so it's ugrizy instead of order in opsim config db.
1332 idx = []
1333 for f in self.filterlist:
1334 filterpos = np.where(propdict['PerFilter']['Filters'] == f)
1335 if len(filterpos[0]) > 0:
1336 idx.append(filterpos[0][0])
1337 idx = np.array(idx, int)
1338 for k in propdict['PerFilter']:
1339 tmp = propdict['PerFilter'][k][idx]
1340 propdict['PerFilter'][k] = tmp
1341 propdict['PerFilter']['keyorder'] = ['Filters', 'VisitTime', 'MaxSeeing', 'MinSky',
1342 'MaxSky', 'NumVisits']
1343 return configSummary, config