Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from __future__ import print_function 

2from builtins import str 

3from builtins import zip 

4import os, re 

5import numpy as np 

6import warnings 

7from .database import Database 

8from lsst.sims.utils import Site 

9from lsst.sims.maf.utils import getDateVersion 

10from sqlalchemy import column 

11 

12__all__ = ['testOpsimVersion', 'OpsimDatabase', 'OpsimDatabaseFBS', 'OpsimDatabaseV4', 'OpsimDatabaseV3'] 

13 

14def testOpsimVersion(database, driver='sqlite', host=None, port=None): 

15 opsdb = Database(database, driver=driver, host=host, port=port) 

16 if 'SummaryAllProps' in opsdb.tableNames: 

17 if 'Field' in opsdb.tableNames: 

18 version = "V4" 

19 else: 

20 version = "FBS" 

21 elif 'Summary' in opsdb.tableNames: 

22 version = "V3" 

23 else: 

24 version = "Unknown" 

25 opsdb.close() 

26 return version 

27 

28def OpsimDatabase(database, driver='sqlite', host=None, port=None, 

29 longstrings=False, verbose=False): 

30 """Convenience method to return an appropriate OpsimDatabaseV3/V4 version. 

31 

32 This is here for backwards compatibility, as 'opsdb = db.OpsimDatabase(dbFile)' will 

33 work as naively expected. However note that OpsimDatabase itself is no longer a class, but 

34 a simple method that will attempt to instantiate the correct type of OpsimDatabaseV3 or OpsimDatabaseV4. 

35 """ 

36 version = testOpsimVersion(database) 

37 if version == 'FBS': 

38 opsdb = OpsimDatabaseFBS(database, driver=driver, host=host, port=port, 

39 longstrings=longstrings, verbose=verbose) 

40 elif version == 'V4': 

41 opsdb = OpsimDatabaseV4(database, driver=driver, host=host, port=port, 

42 longstrings=longstrings, verbose=verbose) 

43 elif version == 'V3': 

44 opsdb = OpsimDatabaseV3(database, driver=driver, host=host, port=port, 

45 longstrings=longstrings, verbose=verbose) 

46 else: 

47 warnings.warn('Could not identify opsim database version; just using Database class instead') 

48 opsdb = Database(database, driver=driver, host=host, port=port, 

49 longstrings=longstrings, verbose=verbose) 

50 return opsdb 

51 

52 

53class BaseOpsimDatabase(Database): 

54 """Base opsim database class to gather common methods among different versions of the opsim schema. 

55 

56 Not intended to be used directly; use OpsimDatabaseFBS, OpsimDatabaseV3 or OpsimDatabaseV4 instead.""" 

57 def __init__(self, database, driver='sqlite', host=None, port=None, defaultTable=None, 

58 longstrings=False, verbose=False): 

59 super(BaseOpsimDatabase, self).__init__(database=database, driver=driver, host=host, port=port, 

60 defaultTable=defaultTable, longstrings=longstrings, 

61 verbose=verbose) 

62 # Save filterlist so that we get the filter info per proposal in this desired order. 

63 self.filterlist = np.array(['u', 'g', 'r', 'i', 'z', 'y']) 

64 self.defaultTable = defaultTable 

65 self._colNames() 

66 

67 def _colNames(self): 

68 # Add version-specific column names in subclasses. 

69 self.opsimVersion = 'unknown' 

70 pass 

71 

72 def fetchMetricData(self, colnames, sqlconstraint=None, groupBy='default', tableName=None): 

73 """ 

74 Fetch 'colnames' from 'tableName'. 

75 

76 Parameters 

77 ---------- 

78 colnames : list 

79 The columns to fetch from the table. 

80 sqlconstraint : str, opt 

81 The sql constraint to apply to the data (minus "WHERE"). Default None. 

82 Examples: to fetch data for the r band filter only, set sqlconstraint to 'filter = "r"'. 

83 groupBy : str, opt 

84 The column to group the returned data by. 

85 Default (when using summaryTable) is the MJD, otherwise will be None. 

86 tableName : str, opt 

87 The table to query. The default (None) will use the summary table, set by self.summaryTable. 

88 

89 Returns 

90 ------- 

91 np.recarray 

92 A structured array containing the data queried from the database. 

93 """ 

94 if tableName is None: 

95 tableName = self.defaultTable 

96 if groupBy == 'default' and tableName==self.defaultTable: 

97 groupBy = self.mjdCol 

98 if groupBy == 'default' and tableName!=self.defaultTable: 

99 groupBy = None 

100 metricdata = super(BaseOpsimDatabase, self).fetchMetricData(colnames=colnames, 

101 sqlconstraint=sqlconstraint, 

102 groupBy=groupBy, tableName=tableName) 

103 return metricdata 

104 

105 def fetchFieldsFromSummaryTable(self, sqlconstraint=None, raColName=None, decColName=None): 

106 """ 

107 Fetch field information (fieldID/RA/Dec) from the summary table. 

108 

109 This implicitly only selects fields which were actually observed by opsim. 

110 

111 Parameters 

112 ---------- 

113 sqlconstraint : str, opt 

114 Sqlconstraint to apply before selecting observations to use for RA/Dec. Default None. 

115 raColName : str, opt 

116 Name of the RA column in the database. 

117 decColName : str, opt 

118 Name of the Dec column in the database. 

119 degreesToRadians : bool, opt 

120 Convert ra/dec into degrees? 

121 If field information in summary table is in degrees, degreesToRadians should be True. 

122 

123 Returns 

124 ------- 

125 np.recarray 

126 Structured array containing the field data (fieldID, fieldRA, fieldDec). RA/Dec in radians. 

127 """ 

128 if raColName is None: 

129 raColName = self.raCol 

130 if decColName is None: 

131 decColName = self.decCol 

132 fielddata = self.query_columns(self.defaultTable, 

133 colnames=[self.fieldIdCol, raColName, decColName], 

134 sqlconstraint=sqlconstraint, groupBy=self.fieldIdCol) 

135 if self.raDecInDeg: 

136 fielddata[raColName] = np.radians(fielddata[raColName]) 

137 fielddata[decColName] = np.radians(fielddata[decColName]) 

138 return fielddata 

139 

140 def fetchRunLength(self): 

141 """Find the survey duration for a particular opsim run (years). 

142 

143 Returns 

144 ------- 

145 float 

146 """ 

147 if 'Config' not in self.tables: 

148 print('Cannot access Config table to retrieve runLength; using default 10 years') 

149 runLength = 10.0 

150 else: 

151 query = 'select paramValue from Config where paramName="%s"' % (self.runLengthParam) 

152 runLength = self.query_arbitrary(query, dtype=[('paramValue', float)]) 

153 runLength = runLength['paramValue'][0] # Years 

154 return runLength 

155 

156 def fetchLatLonHeight(self): 

157 """Returns the latitude, longitude, and height of the telescope used by the config file. 

158 """ 

159 if 'Config' not in self.tables: 

160 print('Cannot access Config table to retrieve site parameters; using sims.utils.Site instead.') 

161 site = Site(name='LSST') 

162 lat = site.latitude_rad 

163 lon = site.longitude_rad 

164 height = site.height 

165 else: 

166 lat = self.query_columns('Config', colnames=['paramValue'], 

167 sqlconstraint="paramName like '%latitude%'") 

168 lat = float(lat['paramValue'][0]) 

169 lon = self.query_columns('Config', colnames=['paramValue'], 

170 sqlconstraint="paramName like '%longitude%'") 

171 lon = float(lon['paramValue'][0]) 

172 height = self.query_columns('Config', colnames=['paramValue'], 

173 sqlconstraint="paramName like '%height%'") 

174 height = float(height['paramValue'][0]) 

175 return lat, lon, height 

176 

177 def fetchOpsimRunName(self): 

178 """Return opsim run name (machine name + session ID) from Session table. 

179 """ 

180 if 'Session' not in self.tables: 

181 print('Could not access Session table to find this information.') 

182 runName = self.defaultRunName 

183 else: 

184 res = self.query_columns('Session', colnames=[self.sessionIdCol, 

185 self.sessionHostCol]) 

186 runName = str(res[self.sessionHostCol][0]) + '_' + str(res[self.sessionIdCol][0]) 

187 return runName 

188 

189 def fetchNVisits(self, propId=None): 

190 """Returns the total number of visits in the simulation or visits for a particular proposal. 

191 

192 Parameters 

193 ---------- 

194 propId : int or list of ints 

195 The ID numbers of the proposal(s). 

196 

197 Returns 

198 ------- 

199 int 

200 """ 

201 if 'ObsHistory' in self.tables and propId is None: 

202 query = 'select count(*) from ObsHistory' 

203 data = self.execute_arbitrary(query, dtype=([('nvisits', int)])) 

204 else: 

205 query = 'select count(distinct(%s)) from %s' %(self.mjdCol, self.defaultTable) 

206 if propId is not None: 

207 query += ' where ' 

208 if isinstance(propId, list) or isinstance(propId, np.ndarray): 

209 for pID in propId: 

210 query += 'propID=%d or ' %(int(pID)) 

211 query = query[:-3] 

212 else: 

213 query += 'propID = %d' %(int(propId)) 

214 data = self.execute_arbitrary(query, dtype=([('nvisits', int)])) 

215 return data['nvisits'][0] 

216 

217 def fetchTotalSlewN(self): 

218 """Return the total number of slews. 

219 """ 

220 if 'SlewActivities' not in self.tables: 

221 print('Could not access SlewActivities table to find this information.') 

222 nslew = -1 

223 else: 

224 query = 'select count(distinct(%s)) from SlewActivities where %s >0' % (self.slewId, 

225 self.delayCol) 

226 res = self.execute_arbitrary(query, dtype=([('slewN', int)])) 

227 nslew = res['slewN'][0] 

228 return nslew 

229 

230 

231class OpsimDatabaseFBS(BaseOpsimDatabase): 

232 """ 

233 Database to class to interact with FBS versions of the opsim outputs. 

234 

235 Parameters 

236 ---------- 

237 database : str 

238 Name of the database or sqlite filename. 

239 driver : str, opt 

240 Name of the dialect + driver for sqlalchemy. Default 'sqlite'. 

241 host : str, opt 

242 Name of the database host. Default None (appropriate for sqlite files). 

243 port : str, opt 

244 String port number for the database. Default None (appropriate for sqlite files). 

245 dbTables : dict, opt 

246 Dictionary of the names of the tables in the database. 

247 The dict should be key = table name, value = [table name, primary key]. 

248 """ 

249 def __init__(self, database, driver='sqlite', host=None, port=None, defaultTable='SummaryAllProps', 

250 longstrings=False, verbose=False): 

251 super().__init__(database=database, driver=driver, host=host, port=port, 

252 defaultTable=defaultTable, longstrings=longstrings, 

253 verbose=verbose) 

254 

255 def _colNames(self): 

256 """ 

257 Set variables to represent the common column names used in this class directly. 

258 

259 This should make future schema changes a little easier to handle. 

260 It is NOT meant to function as a general column map, just to abstract values 

261 which are used *within this class*. 

262 """ 

263 self.mjdCol = column('observationStartMJD') 

264 self.raCol = column('fieldRA') 

265 self.decCol = column('fieldDec') 

266 self.propIdCol = column('proposalId') 

267 # For config parsing. 

268 self.versionCol = 'featureScheduler version' 

269 self.dateCol = 'Date, ymd' 

270 self.raDecInDeg = True 

271 self.opsimVersion = 'FBS' 

272 

273 def fetchPropInfo(self): 

274 """ 

275 There is no inherent proposal information or mapping in the FBS output. 

276 An afterburner script does identify which visits may be counted as contributing toward WFD 

277 and identifies these as such in the proposalID column (0 = general, 1 = WFD, 2+ = DD). 

278 Returns dictionary of propID / propname, and dictionary of propTag / propID. 

279 """ 

280 if 'Proposal' not in self.tables: 

281 print('No proposal table available - no proposalIds have been assigned.') 

282 return {}, {} 

283 

284 propIds = {} 

285 # Add WFD and DD tags by default to propTags as we expect these every time. (avoids key errors). 

286 propTags = {'WFD': [], 'DD': [], 'NES': []} 

287 propData = self.query_columns('Proposal', colnames =[self.propIdCol, 'proposalName', 'proposalType'], 

288 sqlconstraint=None) 

289 for propId, propName, propType in zip(propData[self.propIdCol], 

290 propData['proposalName'], 

291 propData['proposalType']): 

292 propIds[propId] = propName 

293 if propType == 'WFD': 

294 propTags['WFD'].append(propId) 

295 if propType == 'DD': 

296 propTags['DD'].append(propId) 

297 return propIds, propTags 

298 

299 def createSQLWhere(self, tag, propTags): 

300 """ 

301 Create a SQL constraint to identify observations taken for a particular proposal, 

302 using the information in the propTags dictionary. 

303 

304 Parameters 

305 ---------- 

306 tag : str 

307 The name of the proposal for which to create a SQLwhere clause (WFD or DD). 

308 propTags : dict 

309 A dictionary of {proposal name : [proposal ids]} 

310 This can be created using OpsimDatabase.fetchPropInfo() 

311 

312 Returns 

313 ------- 

314 str 

315 The SQL constraint, such as '(proposalID = 1) or (proposalID = 2)' 

316 """ 

317 if (tag not in propTags) or (len(propTags[tag]) == 0): 

318 print('No %s proposals found' % (tag)) 

319 # Create a sqlWhere clause that will not return anything as a query result. 

320 sqlWhere = 'proposalId like "NO PROP"' 

321 elif len(propTags[tag]) == 1: 

322 sqlWhere = "proposalId = %d" % (propTags[tag][0]) 

323 else: 

324 sqlWhere = "(" + " or ".join(["proposalId = %d" % (propid) for propid in propTags[tag]]) + ")" 

325 return sqlWhere 

326 

327 def fetchConfig(self): 

328 """ 

329 Fetch config data from configTable, match proposal IDs with proposal names and some field data, 

330 and do a little manipulation of the data to make it easier to add to the presentation layer. 

331 """ 

332 # Check to see if we're dealing with a full database or not. If not, just return (no config info to fetch). 

333 if 'info' not in self.tables: 

334 warnings.warn('Cannot fetch FBS config info.') 

335 return {}, {} 

336 # Create two dictionaries: a summary dict that contains a summary of the run 

337 configSummary = {} 

338 configSummary['keyorder'] = ['Version', 'RunInfo' ] 

339 # and the other a general dict that contains all the details (by group) of the run. 

340 config = {} 

341 # Start to build up the summary. 

342 # MAF version 

343 mafdate, mafversion = getDateVersion() 

344 configSummary['Version'] = {} 

345 configSummary['Version']['MAFVersion'] = '%s' %(mafversion['__version__']) 

346 configSummary['Version']['MAFDate'] = '%s' %(mafdate) 

347 # Opsim date, version and runcomment info from config info table. 

348 results = self.query_columns('info', ['Parameter', 'Value'], 

349 sqlconstraint=f'Parameter like "{self.versionCol}"') 

350 configSummary['Version']['OpsimVersion'] = 'FBS %s' %(results['Value'][0]) 

351 results = self.query_columns('info', ['Parameter', 'Value'], 

352 sqlconstraint=f'Parameter like "{self.dateCol}"') 

353 opsimdate = '-'.join(results['Value'][0].split(',')).replace(' ', '') 

354 configSummary['Version']['OpsimDate'] = '%s' %(opsimdate) 

355 configSummary['RunInfo'] = {} 

356 results = self.query_columns('info', ['Parameter', 'Value'], 

357 sqlconstraint=f'Parameter like "exec command%"') 

358 configSummary['RunInfo']['Exec'] = '%s' % (results['Value'][0]) 

359 

360 # Echo info table into configDetails. 

361 configDetails = {} 

362 configs = self.query_columns('info', ['Parameter', 'Value']) 

363 for name, value in zip(configs['Parameter'], configs['Value']): 

364 configDetails[name] = value 

365 return configSummary, configDetails 

366 

367 

368class OpsimDatabaseV4(BaseOpsimDatabase): 

369 """ 

370 Database to class to interact with v4 versions of the opsim outputs. 

371 

372 Parameters 

373 ---------- 

374 database : str 

375 Name of the database or sqlite filename. 

376 driver : str, opt 

377 Name of the dialect + driver for sqlalchemy. Default 'sqlite'. 

378 host : str, opt 

379 Name of the database host. Default None (appropriate for sqlite files). 

380 port : str, opt 

381 String port number for the database. Default None (appropriate for sqlite files). 

382 dbTables : dict, opt 

383 Dictionary of the names of the tables in the database. 

384 The dict should be key = table name, value = [table name, primary key]. 

385 """ 

386 def __init__(self, database, driver='sqlite', host=None, port=None, defaultTable='SummaryAllProps', 

387 longstrings=False, verbose=False): 

388 super(OpsimDatabaseV4, self).__init__(database=database, driver=driver, host=host, port=port, 

389 defaultTable=defaultTable, longstrings=longstrings, 

390 verbose=verbose) 

391 

392 def _colNames(self): 

393 """ 

394 Set variables to represent the common column names used in this class directly. 

395 

396 This should make future schema changes a little easier to handle. 

397 It is NOT meant to function as a general column map, just to abstract values 

398 which are used *within this class*. 

399 """ 

400 self.mjdCol = 'observationStartMJD' 

401 self.slewId = 'slewHistory_slewCount' 

402 self.delayCol = 'activityDelay' 

403 self.fieldIdCol = 'fieldId' 

404 self.raCol = 'fieldRA' 

405 self.decCol = 'fieldDec' 

406 self.propIdCol = 'propId' 

407 self.propNameCol = 'propName' 

408 self.propTypeCol = 'propType' 

409 # For config parsing. 

410 self.versionCol = 'version' 

411 self.sessionIdCol = 'sessionId' 

412 self.sessionHostCol = 'sessionHost' 

413 self.sessionDateCol = 'sessionDate' 

414 self.runCommentCol = 'runComment' 

415 self.runLengthParam = 'survey/duration' 

416 self.raDecInDeg = True 

417 self.opsimVersion = 'V4' 

418 

419 def fetchFieldsFromFieldTable(self, propId=None, degreesToRadians=True): 

420 """ 

421 Fetch field information (fieldID/RA/Dec) from the Field table. 

422 

423 This will select fields which were requested by a particular proposal or proposals, 

424 even if they did not receive any observations. 

425 

426 Parameters 

427 ---------- 

428 propId : int or list of ints 

429 Proposal ID or list of proposal IDs to use to select fields. 

430 degreesToRadians : bool, opt 

431 If True, convert degrees in Field table into radians. 

432 

433 Returns 

434 ------- 

435 np.recarray 

436 Structured array containing the field data (fieldID, fieldRA, fieldDec). 

437 """ 

438 if propId is not None: 

439 query = 'select f.fieldId, f.ra, f.dec from Field as f' 

440 query += ', ProposalField as p where (p.Field_fieldId = f.fieldId) ' 

441 if isinstance(propId, list) or isinstance(propId, np.ndarray): 

442 query += ' and (' 

443 for pID in propId: 

444 query += '(p.Proposal_propId = %d) or ' % (int(pID)) 

445 # Remove the trailing 'or' and add a closing parenthesis. 

446 query = query[:-3] 

447 query += ')' 

448 else: # single proposal ID. 

449 query += ' and (p.Proposal_propId = %d) ' %(int(propId)) 

450 query += ' group by f.%s' %(self.fieldIdCol) 

451 fielddata = self.query_arbitrary(query, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol], 

452 ['int', 'float', 'float']))) 

453 if len(fielddata) == 0: 

454 fielddata = np.zeros(0, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol], 

455 ['int', 'float', 'float']))) 

456 else: 

457 query = 'select fieldId, ra, dec from Field group by fieldId' 

458 fielddata = self.query_arbitrary(query, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol], 

459 ['int', 'float', 'float']))) 

460 if len(fielddata) == 0: 

461 fielddata = np.zeros(0, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol], 

462 ['int', 'float', 'float']))) 

463 if degreesToRadians: 

464 fielddata[self.raCol] = fielddata[self.raCol] * np.pi / 180. 

465 fielddata[self.decCol] = fielddata[self.decCol] * np.pi / 180. 

466 return fielddata 

467 

468 

469 def fetchPropInfo(self): 

470 """ 

471 Fetch the proposal IDs as well as their (short) proposal names and science type tags from the 

472 full opsim database. 

473 Returns dictionary of propID / propname, and dictionary of propTag / propID. 

474 If not using a full database, will return dict of propIDs with empty propnames + empty propTag dict. 

475 """ 

476 propIds = {} 

477 # Add WFD and DD tags by default to propTags as we expect these every time. (avoids key errors). 

478 propTags = {'WFD': [], 'DD': [], 'NES': []} 

479 assignData = True 

480 try: 

481 propData = self.query_columns('Proposal', colnames=[self.propIdCol, self.propNameCol], 

482 sqlconstraint=None) 

483 except ValueError: 

484 propData = [] 

485 propIds = {} 

486 propTags = {} 

487 assignData = False 

488 

489 if assignData: 

490 for propId, propName in zip(propData[self.propIdCol], propData[self.propNameCol]): 

491 # Fix these in the future, to use the proper tags that will be added to output database. 

492 propIds[propId] = propName 

493 if 'widefastdeep' in propName.lower(): 

494 propTags['WFD'].append(propId) 

495 if 'drilling' in propName.lower(): 

496 propTags['DD'].append(propId) 

497 if 'northeclipticspur' in propName.lower(): 

498 propTags['NES'].append(propId) 

499 return propIds, propTags 

500 

501 def createSlewConstraint(self, startTime=None, endTime=None): 

502 """Create a SQL constraint for the slew tables (slew activities, slew speeds, slew states) 

503 to select slews between startTime and endTime (MJD). 

504 

505 Parameters 

506 ---------- 

507 startTime : float or None, opt 

508 Time of first slew. Default (None) means no constraint on time of first slew. 

509 endTime : float or None, opt 

510 Time of last slew. Default (None) means no constraint on time of last slew. 

511 

512 Returns 

513 ------- 

514 str 

515 The SQL constraint, like 'slewHistory_slewID between XXX and XXXX' 

516 """ 

517 query = 'select min(observationStartMJD), max(observationStartMJD) from obsHistory' 

518 res = self.query_arbitrary(query, dtype=[('mjdMin', 'int'), ('mjdMax', 'int')]) 

519 # If asking for constraints that are out of bounds of the survey, return None. 

520 if startTime is None or (startTime < res['mjdMin'][0]): 

521 startTime = res['mjdMin'][0] 

522 if endTime is None or (endTime > res['mjdMax'][0]): 

523 endTime = res['mjdMax'][0] 

524 # Check if times were out of bounds. 

525 if startTime > res['mjdMax'][0] or endTime < res['mjdMin'][0]: 

526 warnings.warn('Times requested do not overlap survey operations (%f to %f)' 

527 % (res['mjdMin'][0], res['mjdMax'][0])) 

528 return None 

529 # Find the slew ID matching the start Time. 

530 query = 'select min(observationId) from obsHistory where observationStartMJD >= %s' % (startTime) 

531 res = self.query_arbitrary(query, dtype=[('observationId', 'int')]) 

532 obsHistId = res['observationId'][0] 

533 query = 'select slewCount from slewHistory where ObsHistory_observationId = %d' % (obsHistId) 

534 res = self.query_arbitrary(query, dtype=[('slewCount', 'int')]) 

535 minSlewCount = res['slewCount'][0] 

536 # Find the slew ID matching the end Time. 

537 query = 'select max(observationId) from obsHistory where observationStartMJD <= %s' % (endTime) 

538 res = self.query_arbitrary(query, dtype=[('observationId', 'int')]) 

539 obsHistId = res['observationId'][0] 

540 # Find the observation id corresponding to this slew time. 

541 query = 'select slewCount from slewHistory where ObsHistory_observationId = %d' % (obsHistId) 

542 res = self.query_arbitrary(query, dtype=[('slewCount', 'int')]) 

543 maxSlewCount = res['slewCount'][0] 

544 return 'SlewHistory_slewCount between %d and %d' % (minSlewCount, maxSlewCount) 

545 

546 def createSQLWhere(self, tag, propTags): 

547 """ 

548 Create a SQL constraint to identify observations taken for a particular proposal, 

549 using the information in the propTags dictionary. 

550 

551 Parameters 

552 ---------- 

553 tag : str 

554 The name of the proposal for which to create a SQLwhere clause (WFD or DD). 

555 propTags : dict 

556 A dictionary of {proposal name : [proposal ids]} 

557 This can be created using OpsimDatabase.fetchPropInfo() 

558 

559 Returns 

560 ------- 

561 str 

562 The SQL constraint, such as '(propID = 365) or (propID = 366)' 

563 """ 

564 if (tag not in propTags) or (len(propTags[tag]) == 0): 

565 print('No %s proposals found' % (tag)) 

566 # Create a sqlWhere clause that will not return anything as a query result. 

567 sqlWhere = 'proposalId like "NO PROP"' 

568 elif len(propTags[tag]) == 1: 

569 sqlWhere = "proposalId = %d" % (propTags[tag][0]) 

570 else: 

571 sqlWhere = "(" + " or ".join(["proposalId = %d" % (propid) for propid in propTags[tag]]) + ")" 

572 return sqlWhere 

573 

574 def fetchRequestedNvisits(self, propId=None): 

575 """Find the requested number of visits for the simulation or proposal(s). 

576 

577 Parameters 

578 ---------- 

579 propId : int or list of ints, opt 

580 

581 Returns 

582 ------- 

583 dict 

584 Number of visits in u/g/r/i/z/y. 

585 """ 

586 if propId is None: 

587 constraint = '' 

588 else: 

589 if isinstance(propId, int): 

590 constraint = 'propId = %d' % (propId) 

591 else: 

592 constraint = '' 

593 for pId in propId: 

594 constraint += 'propId = %d or ' % (int(pId)) 

595 constraint = constraint[:-3] 

596 propData = self.query_columns('Proposal', colnames=[self.propNameCol, self.propTypeCol], 

597 sqlconstraint=constraint) 

598 nvisits = {} 

599 for f in self.filterlist: 

600 nvisits[f] = 0 

601 for pName, propType in zip(propData[self.propNameCol], propData[self.propTypeCol]): 

602 if propType.lower() == 'general': 

603 for f in self.filterlist: 

604 constraint = 'paramName="science/general_props/values/%s/filters/%s/num_visits"' \ 

605 % (pName, f) 

606 val = self.query_columns('Config', colnames=['paramValue'], sqlconstraint=constraint) 

607 if len(val) > 0: 

608 nvisits[f] += int(val['paramValue'][0]) 

609 elif propType.lower == 'sequence': 

610 pass 

611 # Not clear yet. 

612 return nvisits 

613 

614 def _matchParamNameValue(self, configarray, keyword): 

615 return configarray['paramValue'][np.where(configarray['paramName'] == keyword)] 

616 

617 def _parseSequences(self, perPropConfig, filterlist): 

618 """ 

619 (Private). Given an array of config paramName/paramValue info for a given WLTSS proposal 

620 and the filterlist of filters used in that proposal, parse the sequences/subsequences and returns: 

621 a dictionary with all the per-sequence information (including subsequence names & events, etc.) 

622 a numpy array with the number of visits per filter 

623 """ 

624 propDict = {} 

625 # Identify where subsequences start in config[propname] arrays. 

626 seqidxs = np.where(perPropConfig['paramName'] == 'SubSeqName')[0] 

627 for sidx in seqidxs: 

628 i = sidx 

629 # Get the name of this subsequence. 

630 seqname = perPropConfig['paramValue'][i] 

631 # Check if seqname is a nested subseq of an existing sequence: 

632 nestedsubseq = False 

633 for prevseq in propDict: 

634 if 'SubSeqNested' in propDict[prevseq]: 

635 if seqname in propDict[prevseq]['SubSeqNested']: 

636 seqdict = propDict[prevseq]['SubSeqNested'][seqname] 

637 nestedsubseq = True 

638 # If not, then create a new subseqence key/dictionary for this subseq. 

639 if not nestedsubseq: 

640 propDict[seqname] = {} 

641 seqdict = propDict[seqname] 

642 # And move on to next parameters within subsequence set. 

643 i += 1 

644 if perPropConfig['paramName'][i] == 'SubSeqNested': 

645 subseqnestedname = perPropConfig['paramValue'][i] 

646 if subseqnestedname != '.': 

647 # Have nested subsequence, so keep track of that here 

648 # but will fill in info later. 

649 seqdict['SubSeqNested'] = {} 

650 # Set up nested dictionary for nested subsequence. 

651 seqdict['SubSeqNested'][subseqnestedname] = {} 

652 i += 1 

653 subseqfilters = perPropConfig['paramValue'][i] 

654 if subseqfilters != '.': 

655 seqdict['Filters'] = subseqfilters 

656 i += 1 

657 subseqexp = perPropConfig['paramValue'][i] 

658 if subseqexp != '.': 

659 seqdict['SubSeqExp'] = subseqexp 

660 i+= 1 

661 subseqevents = perPropConfig['paramValue'][i] 

662 seqdict['Events'] = int(subseqevents) 

663 i+=2 

664 subseqinterval = perPropConfig['paramValue'][i] 

665 subseqint = np.array([subseqinterval.split('*')], 'float').prod() 

666 # In days .. 

667 subseqint *= 1/24.0/60.0/60.0 

668 if subseqint > 1: 

669 seqdict['SubSeqInt'] = '%.2f days' %(subseqint) 

670 else: 

671 subseqint *= 24.0 

672 if subseqint > 1: 

673 seqdict['SubSeqInt'] = '%.2f hours' %(subseqint) 

674 else: 

675 subseqint *= 60.0 

676 seqdict['SubSeqInt'] = '%.3f minutes' %(subseqint) 

677 # End of assigning subsequence info - move on to counting number of visits. 

678 nvisits = np.zeros(len(filterlist), int) 

679 for subseq in propDict: 

680 subevents = propDict[subseq]['Events'] 

681 # Count visits from direct subsequences. 

682 if 'SubSeqExp' in propDict[subseq] and 'Filters' in propDict[subseq]: 

683 subfilters = propDict[subseq]['Filters'] 

684 subexp = propDict[subseq]['SubSeqExp'] 

685 # If just one filter .. 

686 if len(subfilters) == 1: 

687 idx = np.where(filterlist == subfilters)[0] 

688 nvisits[idx] += subevents * int(subexp) 

689 else: 

690 splitsubfilters = subfilters.split(',') 

691 splitsubexp = subexp.split(',') 

692 for f, exp in zip(splitsubfilters, splitsubexp): 

693 idx = np.where(filterlist == f)[0] 

694 nvisits[idx] += subevents * int(exp) 

695 # Count visits if have nested subsequences. 

696 if 'SubSeqNested' in propDict[subseq]: 

697 for subseqnested in propDict[subseq]['SubSeqNested']: 

698 events = subevents * propDict[subseq]['SubSeqNested'][subseqnested]['Events'] 

699 subfilters = propDict[subseq]['SubSeqNested'][subseqnested]['Filters'] 

700 subexp = propDict[subseq]['SubSeqNested'][subseqnested]['SubSeqExp'] 

701 # If just one filter .. 

702 if len(subfilters) == 1: 

703 idx = np.where(filterlist == subfilters)[0] 

704 nvisits[idx] += events * int(subexp) 

705 # Else may have multiple filters in the subsequence, so must split. 

706 splitsubfilters = subfilters.split(',') 

707 splitsubexp = subexp.split(',') 

708 for f, exp in zip(splitsubfilters, splitsubexp): 

709 idx = np.where(filterlist == f)[0] 

710 nvisits[idx] += int(exp) * events 

711 return propDict, nvisits 

712 

713 def _queryParam(self, constraint): 

714 results = self.query_columns('Config', colnames=['paramValue'], sqlconstraint=constraint) 

715 if len(results) > 0: 

716 return results['paramValue'][0] 

717 else: 

718 return '--' 

719 

720 def fetchConfig(self): 

721 """ 

722 Fetch config data from configTable, match proposal IDs with proposal names and some field data, 

723 and do a little manipulation of the data to make it easier to add to the presentation layer. 

724 """ 

725 # Check to see if we're dealing with a full database or not. If not, just return (no config info to fetch). 

726 if 'Session' not in self.tables: 

727 warnings.warn('Cannot fetch opsim config info as this is not a full opsim database.') 

728 return {}, {} 

729 # Create two dictionaries: a summary dict that contains a summary of the run 

730 configSummary = {} 

731 configSummary['keyorder'] = ['Version', 'RunInfo', 'Proposals'] 

732 # and the other a general dict that contains all the details (by group) of the run. 

733 config = {} 

734 # Start to build up the summary. 

735 # MAF version 

736 mafdate, mafversion = getDateVersion() 

737 configSummary['Version'] = {} 

738 configSummary['Version']['MAFVersion'] = '%s' %(mafversion['__version__']) 

739 configSummary['Version']['MAFDate'] = '%s' %(mafdate) 

740 # Opsim date, version and runcomment info from session table 

741 results = self.query_columns('Session', 

742 [self.versionCol, self.sessionDateCol, self.runCommentCol]) 

743 configSummary['Version']['OpsimVersion'] = '%s' %(results[self.versionCol][0]) 

744 configSummary['Version']['OpsimDate'] = '%s' %(results[self.sessionDateCol][0]) 

745 configSummary['Version']['OpsimDate'] = configSummary['Version']['OpsimDate'][0:10] 

746 configSummary['RunInfo'] = {} 

747 configSummary['RunInfo']['RunComment'] = results[self.runCommentCol][0] 

748 configSummary['RunInfo']['RunName'] = self.fetchOpsimRunName() 

749 # Pull out a few special values to put into summary. 

750 # This section has a number of configuration parameter names hard-coded. 

751 # I've left these here (rather than adding to self_colNames), bc I think schema changes in the config 

752 # files will actually be easier to track here (at least until the opsim configs are cleaned up). 

753 constraint = 'paramName="observatory/telescope/altitude_minpos"' 

754 configSummary['RunInfo']['MinAlt'] = self._queryParam(constraint) 

755 constraint = 'paramName="observatory/telescope/altitude_maxpos"' 

756 configSummary['RunInfo']['MaxAlt'] = self._queryParam(constraint) 

757 constraint = 'paramName="observatory/camera/filter_change_time"' 

758 configSummary['RunInfo']['TimeFilterChange'] = self._queryParam(constraint) 

759 constraint = 'paramName="observatory/camera/readout_time"' 

760 configSummary['RunInfo']['TimeReadout'] = self._queryParam(constraint) 

761 constraint = 'paramName="sched_driver/propboost_weight"' 

762 configSummary['RunInfo']['PropBoostWeight'] = self._queryParam(constraint) 

763 configSummary['RunInfo']['keyorder'] = ['RunName', 'RunComment', 'MinAlt', 'MaxAlt', 

764 'TimeFilterChange', 'TimeReadout', 'PropBoostWeight'] 

765 

766 # Echo config table into configDetails. 

767 configDetails = {} 

768 configs = self.query_columns('Config', ['paramName', 'paramValue']) 

769 for name, value in zip(configs['paramName'], configs['paramValue']): 

770 configDetails[name] = value 

771 

772 # Now finish building the summary to add proposal information. 

773 # Loop through all proposals to add summary information. 

774 propData = self.query_columns('Proposal', [self.propIdCol, self.propNameCol, self.propTypeCol]) 

775 configSummary['Proposals'] = {} 

776 for propid, propname, proptype in zip(propData[self.propIdCol], 

777 propData[self.propNameCol], propData[self.propTypeCol]): 

778 configSummary['Proposals'][propname] = {} 

779 propdict = configSummary['Proposals'][propname] 

780 propdict['keyorder'] = ['PropId', 'PropName', 'PropType', 'Airmass bonus', 'Airmass max', 

781 'HA bonus', 'HA max', 'Time weight', 'Restart Lost Sequences', 

782 'Restart Complete Sequences', 'Filters'] 

783 propdict['PropName'] = propname 

784 propdict['PropId'] = propid 

785 propdict['PropType'] = proptype 

786 # Add some useful information on the proposal parameters. 

787 constraint = 'paramName like "science/%s_props/values/%s/sky_constraints/max_airmass"'\ 

788 % ("%", propname) 

789 propdict['Airmass max'] = self._queryParam(constraint) 

790 constraint = 'paramName like "science/%s_props/values/%s/scheduling/airmass_bonus"'\ 

791 % ("%", propname) 

792 propdict['Airmass bonus'] = self._queryParam(constraint) 

793 constraint = 'paramName like "science/%s_props/values/%s/scheduling/hour_angle_max"'\ 

794 % ("%", propname) 

795 propdict['HA max'] = self._queryParam(constraint) 

796 constraint = 'paramName like "science/%s_props/values/%s/scheduling/hour_angle_bonus"'\ 

797 % ("%", propname) 

798 propdict['HA bonus'] = self._queryParam(constraint) 

799 constraint = 'paramName like "science/%s_props/values/%s/scheduling/time_weight"'\ 

800 % ("%", propname) 

801 propdict['Time weight'] = self._queryParam(constraint) 

802 constraint = 'paramName like "science/%s_props/values/%s/scheduling/restart_lost_sequences"'\ 

803 % ("%", propname) 

804 propdict['Restart Lost Sequences'] = self._queryParam(constraint) 

805 constraint = 'paramName like "science/%s_props/values/%s/scheduling/restart_complete_sequences"'\ 

806 % ("%", propname) 

807 propdict['Restart Complete Sequences'] = self._queryParam(constraint) 

808 # Find number of visits requested per filter for the proposal 

809 # along with min/max sky and airmass values. 

810 propdict['Filters'] = {} 

811 for f in self.filterlist: 

812 propdict['Filters'][f] = {} 

813 propdict['Filters'][f]['Filter'] = f 

814 dictkeys = ['MaxSeeing', 'BrightLimit', 'DarkLimit', 'NumVisits', 'GroupedVisits', 'Snaps'] 

815 querykeys = ['max_seeing', 'bright_limit', 'dark_limit', 'num_visits', 

816 'num_grouped_visits', 'exposures'] 

817 for dk, qk in zip(dictkeys, querykeys): 

818 constraint = 'paramName like "science/%s_props/values/%s/filters/%s/%s"' \ 

819 % ("%", propname, f, qk) 

820 propdict['Filters'][f][dk] = self._queryParam(constraint) 

821 propdict['Filters'][f]['keyorder'] = ['Filter', 'MaxSeeing', 'MinSky', 'MaxSky', 

822 'NumVisits', 'GroupedVisits', 'Snaps'] 

823 propdict['Filters']['keyorder'] = list(self.filterlist) 

824 return configSummary, configDetails 

825 

826 

827class OpsimDatabaseV3(BaseOpsimDatabase): 

828 def __init__(self, database, driver='sqlite', host=None, port=None, defaultTable='Summary', 

829 longstrings=False, verbose=False): 

830 """ 

831 Instantiate object to handle queries of the opsim database. 

832 (In general these will be the sqlite database files produced by opsim, but could 

833 be any database holding those opsim output tables.). 

834 

835 database = Name of database or sqlite filename 

836 driver = Name of database dialect+driver for sqlalchemy (e.g. 'sqlite', 'pymssql+mssql') 

837 host = Name of database host (optional) 

838 port = String port number (optional) 

839 

840 """ 

841 super(OpsimDatabaseV3, self).__init__(database=database, driver=driver, host=host, port=port, 

842 defaultTable=defaultTable, longstrings=longstrings, 

843 verbose=verbose) 

844 

845 def _colNames(self): 

846 """ 

847 Set variables to represent the common column names used in this class directly. 

848 

849 This should make future schema changes a little easier to handle. 

850 It is NOT meant to function as a general column map, just to abstract values 

851 which are used *within this class*. 

852 """ 

853 self.mjdCol = 'expMJD' 

854 self.slewId = 'slewHistory_slewID' 

855 self.delayCol = 'actDelay' 

856 self.fieldIdCol = 'fieldID' 

857 self.raCol = 'fieldRA' 

858 self.decCol = 'fieldDec' 

859 self.propIdCol = 'propID' 

860 self.propConfCol = 'propConf' 

861 self.propNameCol = 'propName' #(propname == proptype) 

862 # For config parsing. 

863 self.versionCol = 'version' 

864 self.sessionIdCol = 'sessionID' 

865 self.sessionHostCol = 'sessionHost' 

866 self.sessionDateCol = 'sessionDate' 

867 self.runCommentCol = 'runComment' 

868 self.runLengthParam = 'nRun' 

869 self.raDecInDeg = False 

870 self.opsimVersion = 'V3' 

871 

872 def fetchFieldsFromFieldTable(self, propId=None, degreesToRadians=True): 

873 """ 

874 Fetch field information (fieldID/RA/Dec) from Field (+Proposal_Field) tables. 

875 

876 propID = the proposal ID (default None), if selecting particular proposal - can be a list 

877 degreesToRadians = RA/Dec values are in degrees in the Field table (so convert to radians). 

878 """ 

879 # Note that you can't select any other sql constraints (such as filter). 

880 # This will select fields which were requested by a particular proposal or proposals, 

881 # even if they didn't get any observations. 

882 if propId is not None: 

883 query = 'select f.%s, f.%s, f.%s from %s as f' %(self.fieldIdCol, self.raCol, self.decCol, 

884 'Field') 

885 query += ', %s as p where (p.Field_%s = f.%s) ' %('Proposal_Field', 

886 self.fieldIdCol, self.fieldIdCol) 

887 if isinstance(propId, list) or isinstance(propId, np.ndarray): 

888 query += ' and (' 

889 for pID in propId: 

890 query += '(p.Proposal_%s = %d) or ' %(self.propIdCol, int(pID)) 

891 # Remove the trailing 'or' and add a closing parenthesis. 

892 query = query[:-3] 

893 query += ')' 

894 else: # single proposal ID. 

895 query += ' and (p.Proposal_%s = %d) ' %(self.propIdCol, int(propId)) 

896 query += ' group by f.%s' %(self.fieldIdCol) 

897 fielddata = self.query_arbitrary(query, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol], 

898 ['int', 'float', 'float']))) 

899 if len(fielddata) == 0: 

900 fielddata = np.zeros(0, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol], 

901 ['int', 'float', 'float']))) 

902 else: 

903 fielddata = self.query_columns('Field', colnames=[self.fieldIdCol, self.raCol, self.decCol], 

904 groupBy = self.fieldIdCol) 

905 if degreesToRadians: 

906 fielddata[self.raCol] = fielddata[self.raCol] * np.pi / 180. 

907 fielddata[self.decCol] = fielddata[self.decCol] * np.pi / 180. 

908 return fielddata 

909 

910 def fetchPropInfo(self): 

911 """ 

912 Fetch the proposal IDs as well as their (short) proposal names and science type tags from the 

913 full opsim database. 

914 Returns dictionary of propID / propname, and dictionary of propTag / propID. 

915 If not using a full database, will return dict of propIDs with empty propnames + empty propTag dict. 

916 """ 

917 propIDs = {} 

918 # Add WFD and DD tags by default to propTags as we expect these every time. (avoids key errors). 

919 propTags = {'WFD':[], 'DD':[], 'NES': []} 

920 # If do not have full database available: 

921 if 'Proposal' not in self.tables: 

922 propData = self.query_columns(self.defaultTable, colnames=[self.propIdCol]) 

923 for propid in propData[self.propIdCol]: 

924 propIDs[int(propid)] = propid 

925 else: 

926 # Query for all propIDs. 

927 propData = self.query_columns('Proposal', colnames=[self.propIdCol, self.propConfCol, 

928 self.propNameCol], sqlconstraint=None) 

929 for propid, propname in zip(propData[self.propIdCol], propData[self.propConfCol]): 

930 # Strip '.conf', 'Prop', and path info. 

931 propIDs[int(propid)] = re.sub('Prop','', re.sub('.conf','', re.sub('.*/', '', propname))) 

932 # Find the 'ScienceType' from the config table, to indicate DD/WFD/Rolling, etc. 

933 sciencetypes = self.query_columns('Config', colnames=['paramValue', 'nonPropID'], 

934 sqlconstraint="paramName like 'ScienceType'") 

935 if len(sciencetypes) == 0: 

936 # Then this was an older opsim run without 'ScienceType' tags, 

937 # so fall back to trying to guess what proposals are WFD or DD. 

938 for propid, propname in propIDs.items(): 

939 if 'universal' in propname.lower(): 

940 propTags['WFD'].append(propid) 

941 if 'deep' in propname.lower(): 

942 propTags['DD'].append(propid) 

943 if 'northecliptic' in propname.lower(): 

944 propTags['NES'].append(propid) 

945 else: 

946 # Newer opsim output with 'ScienceType' fields in conf files. 

947 for sc in sciencetypes: 

948 # ScienceType tag can be multiple values, separated by a ',' 

949 tags = [x.strip(' ') for x in sc['paramValue'].split(',')] 

950 for sciencetype in tags: 

951 if sciencetype in propTags: 

952 propTags[sciencetype].append(int(sc['nonPropID'])) 

953 else: 

954 propTags[sciencetype] = [int(sc['nonPropID']),] 

955 # But even these runs don't tag NES 

956 for propid, propname in propIDs.items(): 

957 if 'northecliptic' in propname.lower(): 

958 propTags['NES'].append(propid) 

959 return propIDs, propTags 

960 

961 def createSlewConstraint(self, startTime=None, endTime=None): 

962 """Create a SQL constraint for the slew tables (slew activities, slew speeds, slew states) 

963 to select slews between startTime and endTime (MJD). 

964 

965 Parameters 

966 ---------- 

967 startTime : float or None, opt 

968 Time of first slew. Default (None) means no constraint on time of first slew. 

969 endTime : float or None, opt 

970 Time of last slew. Default (None) means no constraint on time of last slew. 

971 

972 Returns 

973 ------- 

974 str 

975 The SQL constraint, like 'slewHistory_slewID > XXX and slewHistory_slewID < XXXX' 

976 """ 

977 query = 'select min(expMJD), max(expMJD) from obsHistory' 

978 res = self.query_arbitrary(query, dtype=[('mjdMin', 'int'), ('mjdMax', 'int')]) 

979 # If asking for constraints that are out of bounds of the survey, return None. 

980 if startTime is None or (startTime < res['mjdMin'][0]): 

981 startTime = res['mjdMin'][0] 

982 if endTime is None or (endTime > res['mjdMax'][0]): 

983 endTime = res['mjdMax'][0] 

984 # Check if times were out of bounds. 

985 if startTime > res['mjdMax'][0] or endTime < res['mjdMin'][0]: 

986 warnings.warn('Times requested do not overlap survey operations (%f to %f)' 

987 % (res['mjdMin'][0], res['mjdMax'][0])) 

988 return None 

989 # Find the slew ID matching the start Time. 

990 query = 'select min(obsHistID) from obsHistory where expMJD >= %s' % (startTime) 

991 res = self.query_arbitrary(query, dtype=[('observationId', 'int')]) 

992 obsHistId = res['observationId'][0] 

993 query = 'select slewID from slewHistory where ObsHistory_obsHistID = %d' % (obsHistId) 

994 res = self.query_arbitrary(query, dtype=[('slewCount', 'int')]) 

995 minSlewCount = res['slewCount'][0] 

996 # Find the slew ID matching the end Time. 

997 query = 'select max(obsHistID) from obsHistory where expMJD <= %s' % (endTime) 

998 res = self.query_arbitrary(query, dtype=[('observationId', 'int')]) 

999 obsHistId = res['observationId'][0] 

1000 # Find the observation id corresponding to this slew time. 

1001 query = 'select slewID from slewHistory where ObsHistory_obsHistID = %d' % (obsHistId) 

1002 res = self.query_arbitrary(query, dtype=[('slewCount', 'int')]) 

1003 maxSlewCount = res['slewCount'][0] 

1004 return 'SlewHistory_slewID between %d and %d' % (minSlewCount, maxSlewCount) 

1005 

1006 def createSQLWhere(self, tag, propTags): 

1007 """Create a SQL constraint to identify observations taken for a particular proposal, 

1008 using the information in the propTags dictionary. 

1009 

1010 Parameters 

1011 ---------- 

1012 tag : str 

1013 The name of the proposal for which to create a SQLwhere clause (WFD or DD). 

1014 propTags : dict 

1015 A dictionary of {proposal name : [proposal ids]} 

1016 This can be created using OpsimDatabase.fetchPropInfo() 

1017 

1018 Returns 

1019 ------- 

1020 str 

1021 The SQL constraint, such as '(propID = 365) or (propID = 366)' 

1022 """ 

1023 if (tag not in propTags) or (len(propTags[tag]) == 0): 

1024 print('No %s proposals found' % (tag)) 

1025 # Create a sqlWhere clause that will not return anything as a query result. 

1026 sqlWhere = 'propID like "NO PROP"' 

1027 elif len(propTags[tag]) == 1: 

1028 sqlWhere = "propID = %d" % (propTags[tag][0]) 

1029 else: 

1030 sqlWhere = "(" + " or ".join(["propID = %d" % (propid) for propid in propTags[tag]]) + ")" 

1031 return sqlWhere 

1032 

1033 def fetchRequestedNvisits(self, propId=None): 

1034 """ 

1035 Find the requested number of visits for proposals in propId. 

1036 Returns a dictionary - Nvisits{u/g/r/i/z/y} 

1037 """ 

1038 visitDict = {} 

1039 if propId is None: 

1040 # Get all the available propIds. 

1041 propData = self.query_columns('Proposal', colnames=[self.propIdCol, self.propNameCol], 

1042 sqlconstraint=None) 

1043 else: 

1044 # Get the propType info to go with the propId(s). 

1045 if hasattr(propId, '__iter__'): 

1046 constraint = '(' 

1047 for pi in propId: 

1048 constraint += '(propId = %d) or ' %(pi) 

1049 constraint = constraint[:-4] + ')' 

1050 else: 

1051 constraint = 'propId = %d' %(propId) 

1052 propData = self.query_columns('Proposal', colnames=[self.propIdCol, self.propNameCol], 

1053 sqlconstraint=constraint) 

1054 for pId, propType in zip(propData[self.propIdCol], propData[self.propNameCol]): 

1055 perPropConfig = self.query_columns('Config', colnames=['paramName', 'paramValue'], 

1056 sqlconstraint = 'nonPropID = %d and paramName!="userRegion"' 

1057 %(pId)) 

1058 filterlist = self._matchParamNameValue(perPropConfig, 'Filter') 

1059 if propType == 'WL': 

1060 # For WL proposals, the simple 'Filter_Visits' == the requested number of observations. 

1061 nvisits = np.array(self._matchParamNameValue(perPropConfig, 'Filter_Visits'), int) 

1062 elif propType == 'WLTSS': 

1063 seqDict, nvisits = self._parseSequences(perPropConfig, filterlist) 

1064 visitDict[pId] = {} 

1065 for f, N in zip(filterlist, nvisits): 

1066 visitDict[pId][f] = N 

1067 nvisits = {} 

1068 for f in ['u', 'g', 'r', 'i', 'z', 'y']: 

1069 nvisits[f] = 0 

1070 for pId in visitDict: 

1071 for f in visitDict[pId]: 

1072 nvisits[f] += visitDict[pId][f] 

1073 return nvisits 

1074 

1075 def _matchParamNameValue(self, configarray, keyword): 

1076 return configarray['paramValue'][np.where(configarray['paramName']==keyword)] 

1077 

1078 def _parseSequences(self, perPropConfig, filterlist): 

1079 """ 

1080 (Private). Given an array of config paramName/paramValue info for a given WLTSS proposal 

1081 and the filterlist of filters used in that proposal, parse the sequences/subsequences and returns: 

1082 a dictionary with all the per-sequence information (including subsequence names & events, etc.) 

1083 a numpy array with the number of visits per filter 

1084 """ 

1085 propDict = {} 

1086 # Identify where subsequences start in config[propname] arrays. 

1087 seqidxs = np.where(perPropConfig['paramName'] == 'SubSeqName')[0] 

1088 for sidx in seqidxs: 

1089 i = sidx 

1090 # Get the name of this subsequence. 

1091 seqname = perPropConfig['paramValue'][i] 

1092 # Check if seqname is a nested subseq of an existing sequence: 

1093 nestedsubseq = False 

1094 for prevseq in propDict: 

1095 if 'SubSeqNested' in propDict[prevseq]: 

1096 if seqname in propDict[prevseq]['SubSeqNested']: 

1097 seqdict = propDict[prevseq]['SubSeqNested'][seqname] 

1098 nestedsubseq = True 

1099 # If not, then create a new subseqence key/dictionary for this subseq. 

1100 if not nestedsubseq: 

1101 propDict[seqname] = {} 

1102 seqdict = propDict[seqname] 

1103 # And move on to next parameters within subsequence set. 

1104 i += 1 

1105 if perPropConfig['paramName'][i] == 'SubSeqNested': 

1106 subseqnestedname = perPropConfig['paramValue'][i] 

1107 if subseqnestedname != '.': 

1108 # Have nested subsequence, so keep track of that here 

1109 # but will fill in info later. 

1110 seqdict['SubSeqNested'] = {} 

1111 # Set up nested dictionary for nested subsequence. 

1112 seqdict['SubSeqNested'][subseqnestedname] = {} 

1113 i += 1 

1114 subseqfilters = perPropConfig['paramValue'][i] 

1115 if subseqfilters != '.': 

1116 seqdict['Filters'] = subseqfilters 

1117 i += 1 

1118 subseqexp = perPropConfig['paramValue'][i] 

1119 if subseqexp != '.': 

1120 seqdict['SubSeqExp'] = subseqexp 

1121 i+= 1 

1122 subseqevents = perPropConfig['paramValue'][i] 

1123 seqdict['Events'] = int(subseqevents) 

1124 i+=2 

1125 subseqinterval = perPropConfig['paramValue'][i] 

1126 subseqint = np.array([subseqinterval.split('*')], 'float').prod() 

1127 # In days .. 

1128 subseqint *= 1/24.0/60.0/60.0 

1129 if subseqint > 1: 

1130 seqdict['SubSeqInt'] = '%.2f days' %(subseqint) 

1131 else: 

1132 subseqint *= 24.0 

1133 if subseqint > 1: 

1134 seqdict['SubSeqInt'] = '%.2f hours' %(subseqint) 

1135 else: 

1136 subseqint *= 60.0 

1137 seqdict['SubSeqInt'] = '%.3f minutes' %(subseqint) 

1138 # End of assigning subsequence info - move on to counting number of visits. 

1139 nvisits = np.zeros(len(filterlist), int) 

1140 for subseq in propDict: 

1141 subevents = propDict[subseq]['Events'] 

1142 # Count visits from direct subsequences. 

1143 if 'SubSeqExp' in propDict[subseq] and 'Filters' in propDict[subseq]: 

1144 subfilters = propDict[subseq]['Filters'] 

1145 subexp = propDict[subseq]['SubSeqExp'] 

1146 # If just one filter .. 

1147 if len(subfilters) == 1: 

1148 idx = np.where(filterlist == subfilters)[0] 

1149 nvisits[idx] += subevents * int(subexp) 

1150 else: 

1151 splitsubfilters = subfilters.split(',') 

1152 splitsubexp = subexp.split(',') 

1153 for f, exp in zip(splitsubfilters, splitsubexp): 

1154 idx = np.where(filterlist == f)[0] 

1155 nvisits[idx] += subevents * int(exp) 

1156 # Count visits if have nested subsequences. 

1157 if 'SubSeqNested' in propDict[subseq]: 

1158 for subseqnested in propDict[subseq]['SubSeqNested']: 

1159 events = subevents * propDict[subseq]['SubSeqNested'][subseqnested]['Events'] 

1160 subfilters = propDict[subseq]['SubSeqNested'][subseqnested]['Filters'] 

1161 subexp = propDict[subseq]['SubSeqNested'][subseqnested]['SubSeqExp'] 

1162 # If just one filter .. 

1163 if len(subfilters) == 1: 

1164 idx = np.where(filterlist == subfilters)[0] 

1165 nvisits[idx] += events * int(subexp) 

1166 # Else may have multiple filters in the subsequence, so must split. 

1167 splitsubfilters = subfilters.split(',') 

1168 splitsubexp = subexp.split(',') 

1169 for f, exp in zip(splitsubfilters, splitsubexp): 

1170 idx = np.where(filterlist == f)[0] 

1171 nvisits[idx] += int(exp) * events 

1172 return propDict, nvisits 

1173 

1174 def fetchConfig(self): 

1175 """ 

1176 Fetch config data from configTable, match proposal IDs with proposal names and some field data, 

1177 and do a little manipulation of the data to make it easier to add to the presentation layer. 

1178 """ 

1179 # Check to see if we're dealing with a full database or not. If not, just return (no config info to fetch). 

1180 if 'Session' not in self.tables: 

1181 warnings.warn('Cannot fetch opsim config info as this is not a full opsim database.') 

1182 return {}, {} 

1183 # Create two dictionaries: a summary dict that contains a summary of the run 

1184 configSummary = {} 

1185 configSummary['keyorder'] = ['Version', 'RunInfo', 'Proposals'] 

1186 # and the other a general dict that contains all the details (by group) of the run. 

1187 config = {} 

1188 # Start to build up the summary. 

1189 # MAF version 

1190 mafdate, mafversion = getDateVersion() 

1191 configSummary['Version'] = {} 

1192 configSummary['Version']['MAFVersion'] = '%s' % (mafversion['__version__']) 

1193 configSummary['Version']['MAFDate'] = '%s' % (mafdate) 

1194 # Opsim date, version and runcomment info from session table 

1195 results = self.query_columns('Session', colnames = [self.versionCol, self.sessionDateCol, 

1196 self.runCommentCol]) 

1197 configSummary['Version']['OpsimVersion'] = '%s' % (results[self.versionCol][0]) 

1198 configSummary['Version']['OpsimDate'] = '%s' % (results[self.sessionDateCol][0]) 

1199 configSummary['RunInfo'] = {} 

1200 configSummary['RunInfo']['RunComment'] = results[self.runCommentCol] 

1201 configSummary['RunInfo']['RunName'] = self.fetchOpsimRunName() 

1202 # Pull out a few special values to put into summary. 

1203 # This section has a number of configuration parameter names hard-coded. 

1204 # I've left these here (rather than adding to self_colNames), because I think schema changes will in the config 

1205 # files will actually be easier to track here (at least until the opsim configs are cleaned up). 

1206 constraint = 'moduleName="Config" and paramName="sha1"' 

1207 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint) 

1208 try: 

1209 configSummary['Version']['Config sha1'] = results['paramValue'][0] 

1210 except IndexError: 

1211 configSummary['Version']['Config sha1'] = 'Unavailable' 

1212 constraint = 'moduleName="Config" and paramName="changedFiles"' 

1213 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint) 

1214 try: 

1215 configSummary['Version']['Config changed files'] = results['paramValue'][0] 

1216 except IndexError: 

1217 configSummary['Version']['Config changed files'] = 'Unavailable' 

1218 constraint = 'moduleName="instrument" and paramName="Telescope_AltMin"' 

1219 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint) 

1220 configSummary['RunInfo']['MinAlt'] = results['paramValue'][0] 

1221 constraint = 'moduleName="instrument" and paramName="Telescope_AltMax"' 

1222 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint) 

1223 configSummary['RunInfo']['MaxAlt'] = results['paramValue'][0] 

1224 constraint = 'moduleName="instrument" and paramName="Filter_MoveTime"' 

1225 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint) 

1226 configSummary['RunInfo']['TimeFilterChange'] = results['paramValue'][0] 

1227 constraint = 'moduleName="instrument" and paramName="Readout_Time"' 

1228 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint) 

1229 configSummary['RunInfo']['TimeReadout'] = results['paramValue'][0] 

1230 constraint = 'moduleName="scheduler" and paramName="MinDistance2Moon"' 

1231 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint) 

1232 configSummary['RunInfo']['MinDist2Moon'] = results['paramValue'][0] 

1233 configSummary['RunInfo']['keyorder'] = ['RunName', 'RunComment', 'MinDist2Moon', 'MinAlt', 'MaxAlt', 

1234 'TimeFilterChange', 'TimeReadout'] 

1235 # Now build up config dict with 'nice' group names (proposal name and short module name) 

1236 # Each dict entry is a numpy array with the paramName/paramValue/comment values. 

1237 # Match proposal IDs with names. 

1238 query = 'select %s, %s, %s from Proposal group by %s' %(self.propIdCol, self.propConfCol, 

1239 self.propNameCol, self.propIdCol) 

1240 propdata = self.query_arbitrary(query, dtype=([(self.propIdCol, int), 

1241 (self.propConfCol, np.str_, 256), 

1242 (self.propNameCol, np.str_, 256)])) 

1243 # Make 'nice' proposal names 

1244 propnames = np.array([os.path.split(x)[1].replace('.conf', '') for x in propdata[self.propConfCol]]) 

1245 # Get 'nice' module names 

1246 moduledata = self.query_columns('Config', colnames=['moduleName',], sqlconstraint='nonPropID=0') 

1247 modulenames = np.array([os.path.split(x)[1].replace('.conf', '') for x in moduledata['moduleName']]) 

1248 # Grab the config information for each proposal and module. 

1249 cols = ['paramName', 'paramValue', 'comment'] 

1250 for longmodname, modname in zip(moduledata['moduleName'], modulenames): 

1251 config[modname] = self.query_columns('Config', colnames=cols, 

1252 sqlconstraint='moduleName="%s"' %(longmodname)) 

1253 config[modname] = config[modname][['paramName', 'paramValue', 'comment']] 

1254 for propid, propname in zip(propdata[self.propIdCol], propnames): 

1255 config[propname] = self.query_columns('Config', colnames=cols, 

1256 sqlconstraint='nonPropID="%s" and paramName!="userRegion"' 

1257 %(propid)) 

1258 config[propname] = config[propname][['paramName', 'paramValue', 'comment']] 

1259 config['keyorder'] = ['Comment', 'LSST', 'site', 'instrument', 'filters', 

1260 'AstronomicalSky', 'File', 'scheduler', 

1261 'schedulingData', 'schedDown', 'unschedDown'] 

1262 # Now finish building the summary to add proposal information. 

1263 # Loop through all proposals to add summary information. 

1264 configSummary['Proposals'] = {} 

1265 propidorder = sorted(propdata[self.propIdCol]) 

1266 # Generate a keyorder to print proposals in order of propid. 

1267 configSummary['Proposals']['keyorder'] = [] 

1268 for propid in propidorder: 

1269 configSummary['Proposals']['keyorder'].append(propnames[np.where(propdata[self.propIdCol] 

1270 == propid)][0]) 

1271 for propid, propname in zip(propdata[self.propIdCol], propnames): 

1272 configSummary['Proposals'][propname] = {} 

1273 propdict = configSummary['Proposals'][propname] 

1274 propdict['keyorder'] = [self.propIdCol, self.propNameCol, 'PropType', 'RelPriority', 

1275 'NumUserRegions', 'NumFields'] 

1276 propdict[self.propNameCol] = propname 

1277 propdict[self.propIdCol] = propid 

1278 propdict['PropType'] = propdata[self.propNameCol][np.where(propnames == propname)] 

1279 propdict['RelPriority'] = self._matchParamNameValue(config[propname], 'RelativeProposalPriority') 

1280 # Get the number of user regions. 

1281 constraint = 'nonPropID="%s" and paramName="userRegion"' %(propid) 

1282 result = self.query_columns('Config', colnames=['paramName',], sqlconstraint=constraint) 

1283 propdict['NumUserRegions'] = result.size 

1284 # Get the number of fields requested in the proposal (all filters). 

1285 propdict['NumFields'] = self.fetchFieldsFromFieldTable(propId=propid).size 

1286 # Find number of visits requested per filter for the proposal, with min/max sky & airmass values. 

1287 # Note that config table has multiple entries for Filter/Filter_Visits/etc. with the same name. 

1288 # The order of these entries in the config array matters. 

1289 propdict['PerFilter'] = {} 

1290 for key, keyword in zip(['Filters', 'MaxSeeing', 'MinSky', 'MaxSky'], 

1291 ['Filter', 'Filter_MaxSeeing', 'Filter_MinBrig', 'Filter_MaxBrig']): 

1292 temp = self._matchParamNameValue(config[propname], keyword) 

1293 if len(temp) > 0: 

1294 propdict['PerFilter'][key] = temp 

1295 # Add exposure time, potentially looking for scaling per filter. 

1296 exptime = float(self._matchParamNameValue(config[propname], 'ExposureTime')[0]) 

1297 temp = self._matchParamNameValue(config[propname], 'Filter_ExpFactor') 

1298 if len(temp) > 0: 

1299 propdict['PerFilter']['VisitTime'] = temp * exptime 

1300 else: 

1301 propdict['PerFilter']['VisitTime'] = np.ones(len(propdict['PerFilter']['Filters']), float) 

1302 propdict['PerFilter']['VisitTime'] *= exptime 

1303 # And count how many total exposures are requested per filter. 

1304 # First check if 'RestartCompleteSequences' is true: 

1305 # if both are true, then basically an indefinite number of visits are requested, although we're 

1306 # not going to count this way (as the proposals still make an approximate number of requests). 

1307 restartComplete = False 

1308 temp = self._matchParamNameValue(config[propname], 'RestartCompleteSequences') 

1309 if len(temp) > 0: 

1310 if temp[0] == 'True': 

1311 restartComplete = True 

1312 propdict['RestartCompleteSequences'] = restartComplete 

1313 # Grab information on restarting lost sequences so we can print this too. 

1314 restartLost = False 

1315 tmp = self._matchParamNameValue(config[propname], 'RestartLostSequences') 

1316 if len(temp) > 0: 

1317 if temp[0] == 'True': 

1318 restartLost = True 

1319 propdict['RestartLostSequences'] = restartLost 

1320 if propdict['PropType'] == 'WL': 

1321 # Simple 'Filter_Visits' request for number of observations. 

1322 propdict['PerFilter']['NumVisits'] = np.array(self._matchParamNameValue(config[propname], 

1323 'Filter_Visits'), int) 

1324 elif propdict['PropType'] == 'WLTSS': 

1325 # Proposal contains subsequences and possible nested subseq, so must delve further. 

1326 # Make a dictionary to hold the subsequence info (keyed per subsequence). 

1327 propdict['SubSeq'], Nvisits = self._parseSequences(config[propname], 

1328 propdict['PerFilter']['Filters']) 

1329 propdict['PerFilter']['NumVisits'] = Nvisits 

1330 propdict['SubSeq']['keyorder'] = ['SubSeqName', 'SubSeqNested', 'Events', 'SubSeqInt'] 

1331 # Sort the filter information so it's ugrizy instead of order in opsim config db. 

1332 idx = [] 

1333 for f in self.filterlist: 

1334 filterpos = np.where(propdict['PerFilter']['Filters'] == f) 

1335 if len(filterpos[0]) > 0: 

1336 idx.append(filterpos[0][0]) 

1337 idx = np.array(idx, int) 

1338 for k in propdict['PerFilter']: 

1339 tmp = propdict['PerFilter'][k][idx] 

1340 propdict['PerFilter'][k] = tmp 

1341 propdict['PerFilter']['keyorder'] = ['Filters', 'VisitTime', 'MaxSeeing', 'MinSky', 

1342 'MaxSky', 'NumVisits'] 

1343 return configSummary, config 

1344