Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1from __future__ import print_function 

2from builtins import str 

3from builtins import zip 

4import os, re 

5import numpy as np 

6import warnings 

7from .database import Database 

8from lsst.sims.utils import Site 

9from lsst.sims.maf.utils import getDateVersion 

10 

11__all__ = ['testOpsimVersion', 'OpsimDatabase', 'OpsimDatabaseFBS', 'OpsimDatabaseV4', 'OpsimDatabaseV3'] 

12 

13def testOpsimVersion(database, driver='sqlite', host=None, port=None): 

14 opsdb = Database(database, driver=driver, host=host, port=port) 

15 if 'SummaryAllProps' in opsdb.tableNames: 

16 if 'Field' in opsdb.tableNames: 

17 version = "V4" 

18 else: 

19 version = "FBS" 

20 elif 'Summary' in opsdb.tableNames: 

21 version = "V3" 

22 else: 

23 version = "Unknown" 

24 opsdb.close() 

25 return version 

26 

27def OpsimDatabase(database, driver='sqlite', host=None, port=None, 

28 longstrings=False, verbose=False): 

29 """Convenience method to return an appropriate OpsimDatabaseV3/V4 version. 

30 

31 This is here for backwards compatibility, as 'opsdb = db.OpsimDatabase(dbFile)' will 

32 work as naively expected. However note that OpsimDatabase itself is no longer a class, but 

33 a simple method that will attempt to instantiate the correct type of OpsimDatabaseV3 or OpsimDatabaseV4. 

34 """ 

35 version = testOpsimVersion(database) 

36 if version == 'FBS': 

37 opsdb = OpsimDatabaseFBS(database, driver=driver, host=host, port=port, 

38 longstrings=longstrings, verbose=verbose) 

39 elif version == 'V4': 

40 opsdb = OpsimDatabaseV4(database, driver=driver, host=host, port=port, 

41 longstrings=longstrings, verbose=verbose) 

42 elif version == 'V3': 

43 opsdb = OpsimDatabaseV3(database, driver=driver, host=host, port=port, 

44 longstrings=longstrings, verbose=verbose) 

45 else: 

46 warnings.warn('Could not identify opsim database version; just using Database class instead') 

47 opsdb = Database(database, driver=driver, host=host, port=port, 

48 longstrings=longstrings, verbose=verbose) 

49 return opsdb 

50 

51 

52class BaseOpsimDatabase(Database): 

53 """Base opsim database class to gather common methods among different versions of the opsim schema. 

54 

55 Not intended to be used directly; use OpsimDatabaseFBS, OpsimDatabaseV3 or OpsimDatabaseV4 instead.""" 

56 def __init__(self, database, driver='sqlite', host=None, port=None, defaultTable=None, 

57 longstrings=False, verbose=False): 

58 super(BaseOpsimDatabase, self).__init__(database=database, driver=driver, host=host, port=port, 

59 defaultTable=defaultTable, longstrings=longstrings, 

60 verbose=verbose) 

61 # Save filterlist so that we get the filter info per proposal in this desired order. 

62 self.filterlist = np.array(['u', 'g', 'r', 'i', 'z', 'y']) 

63 self.defaultTable = defaultTable 

64 self._colNames() 

65 

66 def _colNames(self): 

67 # Add version-specific column names in subclasses. 

68 self.opsimVersion = 'unknown' 

69 pass 

70 

71 def fetchMetricData(self, colnames, sqlconstraint=None, groupBy='default', tableName=None): 

72 """ 

73 Fetch 'colnames' from 'tableName'. 

74 

75 Parameters 

76 ---------- 

77 colnames : list 

78 The columns to fetch from the table. 

79 sqlconstraint : str, opt 

80 The sql constraint to apply to the data (minus "WHERE"). Default None. 

81 Examples: to fetch data for the r band filter only, set sqlconstraint to 'filter = "r"'. 

82 groupBy : str, opt 

83 The column to group the returned data by. 

84 Default (when using summaryTable) is the MJD, otherwise will be None. 

85 tableName : str, opt 

86 The table to query. The default (None) will use the summary table, set by self.summaryTable. 

87 

88 Returns 

89 ------- 

90 np.recarray 

91 A structured array containing the data queried from the database. 

92 """ 

93 if tableName is None: 

94 tableName = self.defaultTable 

95 if groupBy is 'default' and tableName==self.defaultTable: 

96 groupBy = self.mjdCol 

97 if groupBy is 'default' and tableName!=self.defaultTable: 

98 groupBy = None 

99 metricdata = super(BaseOpsimDatabase, self).fetchMetricData(colnames=colnames, 

100 sqlconstraint=sqlconstraint, 

101 groupBy=groupBy, tableName=tableName) 

102 return metricdata 

103 

104 def fetchFieldsFromSummaryTable(self, sqlconstraint=None, raColName=None, decColName=None): 

105 """ 

106 Fetch field information (fieldID/RA/Dec) from the summary table. 

107 

108 This implicitly only selects fields which were actually observed by opsim. 

109 

110 Parameters 

111 ---------- 

112 sqlconstraint : str, opt 

113 Sqlconstraint to apply before selecting observations to use for RA/Dec. Default None. 

114 raColName : str, opt 

115 Name of the RA column in the database. 

116 decColName : str, opt 

117 Name of the Dec column in the database. 

118 degreesToRadians : bool, opt 

119 Convert ra/dec into degrees? 

120 If field information in summary table is in degrees, degreesToRadians should be True. 

121 

122 Returns 

123 ------- 

124 np.recarray 

125 Structured array containing the field data (fieldID, fieldRA, fieldDec). RA/Dec in radians. 

126 """ 

127 if raColName is None: 

128 raColName = self.raCol 

129 if decColName is None: 

130 decColName = self.decCol 

131 fielddata = self.query_columns(self.defaultTable, 

132 colnames=[self.fieldIdCol, raColName, decColName], 

133 sqlconstraint=sqlconstraint, groupBy=self.fieldIdCol) 

134 if self.raDecInDeg: 

135 fielddata[raColName] = np.radians(fielddata[raColName]) 

136 fielddata[decColName] = np.radians(fielddata[decColName]) 

137 return fielddata 

138 

139 def fetchRunLength(self): 

140 """Find the survey duration for a particular opsim run (years). 

141 

142 Returns 

143 ------- 

144 float 

145 """ 

146 if 'Config' not in self.tables: 

147 print('Cannot access Config table to retrieve runLength; using default 10 years') 

148 runLength = 10.0 

149 else: 

150 query = 'select paramValue from Config where paramName="%s"' % (self.runLengthParam) 

151 runLength = self.query_arbitrary(query, dtype=[('paramValue', float)]) 

152 runLength = runLength['paramValue'][0] # Years 

153 return runLength 

154 

155 def fetchLatLonHeight(self): 

156 """Returns the latitude, longitude, and height of the telescope used by the config file. 

157 """ 

158 if 'Config' not in self.tables: 

159 print('Cannot access Config table to retrieve site parameters; using sims.utils.Site instead.') 

160 site = Site(name='LSST') 

161 lat = site.latitude_rad 

162 lon = site.longitude_rad 

163 height = site.height 

164 else: 

165 lat = self.query_columns('Config', colnames=['paramValue'], 

166 sqlconstraint="paramName like '%latitude%'") 

167 lat = float(lat['paramValue'][0]) 

168 lon = self.query_columns('Config', colnames=['paramValue'], 

169 sqlconstraint="paramName like '%longitude%'") 

170 lon = float(lon['paramValue'][0]) 

171 height = self.query_columns('Config', colnames=['paramValue'], 

172 sqlconstraint="paramName like '%height%'") 

173 height = float(height['paramValue'][0]) 

174 return lat, lon, height 

175 

176 def fetchOpsimRunName(self): 

177 """Return opsim run name (machine name + session ID) from Session table. 

178 """ 

179 if 'Session' not in self.tables: 

180 print('Could not access Session table to find this information.') 

181 runName = self.defaultRunName 

182 else: 

183 res = self.query_columns('Session', colnames=[self.sessionIdCol, 

184 self.sessionHostCol]) 

185 runName = str(res[self.sessionHostCol][0]) + '_' + str(res[self.sessionIdCol][0]) 

186 return runName 

187 

188 def fetchNVisits(self, propId=None): 

189 """Returns the total number of visits in the simulation or visits for a particular proposal. 

190 

191 Parameters 

192 ---------- 

193 propId : int or list of ints 

194 The ID numbers of the proposal(s). 

195 

196 Returns 

197 ------- 

198 int 

199 """ 

200 if 'ObsHistory' in self.tables and propId is None: 

201 query = 'select count(*) from ObsHistory' 

202 data = self.execute_arbitrary(query, dtype=([('nvisits', int)])) 

203 else: 

204 query = 'select count(distinct(%s)) from %s' %(self.mjdCol, self.defaultTable) 

205 if propId is not None: 

206 query += ' where ' 

207 if isinstance(propId, list) or isinstance(propId, np.ndarray): 

208 for pID in propId: 

209 query += 'propID=%d or ' %(int(pID)) 

210 query = query[:-3] 

211 else: 

212 query += 'propID = %d' %(int(propId)) 

213 data = self.execute_arbitrary(query, dtype=([('nvisits', int)])) 

214 return data['nvisits'][0] 

215 

216 def fetchTotalSlewN(self): 

217 """Return the total number of slews. 

218 """ 

219 if 'SlewActivities' not in self.tables: 

220 print('Could not access SlewActivities table to find this information.') 

221 nslew = -1 

222 else: 

223 query = 'select count(distinct(%s)) from SlewActivities where %s >0' % (self.slewId, 

224 self.delayCol) 

225 res = self.execute_arbitrary(query, dtype=([('slewN', int)])) 

226 nslew = res['slewN'][0] 

227 return nslew 

228 

229 

230class OpsimDatabaseFBS(BaseOpsimDatabase): 

231 """ 

232 Database to class to interact with FBS versions of the opsim outputs. 

233 

234 Parameters 

235 ---------- 

236 database : str 

237 Name of the database or sqlite filename. 

238 driver : str, opt 

239 Name of the dialect + driver for sqlalchemy. Default 'sqlite'. 

240 host : str, opt 

241 Name of the database host. Default None (appropriate for sqlite files). 

242 port : str, opt 

243 String port number for the database. Default None (appropriate for sqlite files). 

244 dbTables : dict, opt 

245 Dictionary of the names of the tables in the database. 

246 The dict should be key = table name, value = [table name, primary key]. 

247 """ 

248 def __init__(self, database, driver='sqlite', host=None, port=None, defaultTable='SummaryAllProps', 

249 longstrings=False, verbose=False): 

250 super().__init__(database=database, driver=driver, host=host, port=port, 

251 defaultTable=defaultTable, longstrings=longstrings, 

252 verbose=verbose) 

253 

254 def _colNames(self): 

255 """ 

256 Set variables to represent the common column names used in this class directly. 

257 

258 This should make future schema changes a little easier to handle. 

259 It is NOT meant to function as a general column map, just to abstract values 

260 which are used *within this class*. 

261 """ 

262 self.mjdCol = 'observationStartMJD' 

263 self.raCol = 'fieldRA' 

264 self.decCol = 'fieldDec' 

265 self.propIdCol = 'proposalId' 

266 # For config parsing. 

267 self.versionCol = 'featureScheduler version' 

268 self.dateCol = 'Date, ymd' 

269 self.raDecInDeg = True 

270 self.opsimVersion = 'FBS' 

271 

272 def fetchPropInfo(self): 

273 """ 

274 There is no inherent proposal information or mapping in the FBS output. 

275 An afterburner script does identify which visits may be counted as contributing toward WFD 

276 and identifies these as such in the proposalID column (0 = general, 1 = WFD, 2+ = DD). 

277 Returns dictionary of propID / propname, and dictionary of propTag / propID. 

278 """ 

279 if 'Proposal' not in self.tables: 

280 print('No proposal table available - no proposalIds have been assigned.') 

281 return {}, {} 

282 

283 propIds = {} 

284 # Add WFD and DD tags by default to propTags as we expect these every time. (avoids key errors). 

285 propTags = {'WFD': [], 'DD': [], 'NES': []} 

286 propData = self.query_columns('Proposal', colnames =[self.propIdCol, 'proposalName', 'proposalType'], 

287 sqlconstraint=None) 

288 for propId, propName, propType in zip(propData[self.propIdCol], 

289 propData['proposalName'], 

290 propData['proposalType']): 

291 propIds[propId] = propName 

292 if propType == 'WFD': 

293 propTags['WFD'].append(propId) 

294 if propType == 'DD': 

295 propTags['DD'].append(propId) 

296 return propIds, propTags 

297 

298 def createSQLWhere(self, tag, propTags): 

299 """ 

300 Create a SQL constraint to identify observations taken for a particular proposal, 

301 using the information in the propTags dictionary. 

302 

303 Parameters 

304 ---------- 

305 tag : str 

306 The name of the proposal for which to create a SQLwhere clause (WFD or DD). 

307 propTags : dict 

308 A dictionary of {proposal name : [proposal ids]} 

309 This can be created using OpsimDatabase.fetchPropInfo() 

310 

311 Returns 

312 ------- 

313 str 

314 The SQL constraint, such as '(proposalID = 1) or (proposalID = 2)' 

315 """ 

316 if (tag not in propTags) or (len(propTags[tag]) == 0): 

317 print('No %s proposals found' % (tag)) 

318 # Create a sqlWhere clause that will not return anything as a query result. 

319 sqlWhere = 'proposalId like "NO PROP"' 

320 elif len(propTags[tag]) == 1: 

321 sqlWhere = "proposalId = %d" % (propTags[tag][0]) 

322 else: 

323 sqlWhere = "(" + " or ".join(["proposalId = %d" % (propid) for propid in propTags[tag]]) + ")" 

324 return sqlWhere 

325 

326 def fetchConfig(self): 

327 """ 

328 Fetch config data from configTable, match proposal IDs with proposal names and some field data, 

329 and do a little manipulation of the data to make it easier to add to the presentation layer. 

330 """ 

331 # Check to see if we're dealing with a full database or not. If not, just return (no config info to fetch). 

332 if 'info' not in self.tables: 

333 warnings.warn('Cannot fetch FBS config info.') 

334 return {}, {} 

335 # Create two dictionaries: a summary dict that contains a summary of the run 

336 configSummary = {} 

337 configSummary['keyorder'] = ['Version', 'RunInfo' ] 

338 # and the other a general dict that contains all the details (by group) of the run. 

339 config = {} 

340 # Start to build up the summary. 

341 # MAF version 

342 mafdate, mafversion = getDateVersion() 

343 configSummary['Version'] = {} 

344 configSummary['Version']['MAFVersion'] = '%s' %(mafversion['__version__']) 

345 configSummary['Version']['MAFDate'] = '%s' %(mafdate) 

346 # Opsim date, version and runcomment info from config info table. 

347 results = self.query_columns('info', ['Parameter', 'Value'], 

348 sqlconstraint=f'Parameter like "{self.versionCol}"') 

349 configSummary['Version']['OpsimVersion'] = 'FBS %s' %(results['Value'][0]) 

350 results = self.query_columns('info', ['Parameter', 'Value'], 

351 sqlconstraint=f'Parameter like "{self.dateCol}"') 

352 opsimdate = '-'.join(results['Value'][0].split(',')).replace(' ', '') 

353 configSummary['Version']['OpsimDate'] = '%s' %(opsimdate) 

354 configSummary['RunInfo'] = {} 

355 results = self.query_columns('info', ['Parameter', 'Value'], 

356 sqlconstraint=f'Parameter like "exec command%"') 

357 configSummary['RunInfo']['Exec'] = '%s' % (results['Value'][0]) 

358 

359 # Echo info table into configDetails. 

360 configDetails = {} 

361 configs = self.query_columns('info', ['Parameter', 'Value']) 

362 for name, value in zip(configs['Parameter'], configs['Value']): 

363 configDetails[name] = value 

364 return configSummary, configDetails 

365 

366 

367class OpsimDatabaseV4(BaseOpsimDatabase): 

368 """ 

369 Database to class to interact with v4 versions of the opsim outputs. 

370 

371 Parameters 

372 ---------- 

373 database : str 

374 Name of the database or sqlite filename. 

375 driver : str, opt 

376 Name of the dialect + driver for sqlalchemy. Default 'sqlite'. 

377 host : str, opt 

378 Name of the database host. Default None (appropriate for sqlite files). 

379 port : str, opt 

380 String port number for the database. Default None (appropriate for sqlite files). 

381 dbTables : dict, opt 

382 Dictionary of the names of the tables in the database. 

383 The dict should be key = table name, value = [table name, primary key]. 

384 """ 

385 def __init__(self, database, driver='sqlite', host=None, port=None, defaultTable='SummaryAllProps', 

386 longstrings=False, verbose=False): 

387 super(OpsimDatabaseV4, self).__init__(database=database, driver=driver, host=host, port=port, 

388 defaultTable=defaultTable, longstrings=longstrings, 

389 verbose=verbose) 

390 

391 def _colNames(self): 

392 """ 

393 Set variables to represent the common column names used in this class directly. 

394 

395 This should make future schema changes a little easier to handle. 

396 It is NOT meant to function as a general column map, just to abstract values 

397 which are used *within this class*. 

398 """ 

399 self.mjdCol = 'observationStartMJD' 

400 self.slewId = 'slewHistory_slewCount' 

401 self.delayCol = 'activityDelay' 

402 self.fieldIdCol = 'fieldId' 

403 self.raCol = 'fieldRA' 

404 self.decCol = 'fieldDec' 

405 self.propIdCol = 'propId' 

406 self.propNameCol = 'propName' 

407 self.propTypeCol = 'propType' 

408 # For config parsing. 

409 self.versionCol = 'version' 

410 self.sessionIdCol = 'sessionId' 

411 self.sessionHostCol = 'sessionHost' 

412 self.sessionDateCol = 'sessionDate' 

413 self.runCommentCol = 'runComment' 

414 self.runLengthParam = 'survey/duration' 

415 self.raDecInDeg = True 

416 self.opsimVersion = 'V4' 

417 

418 def fetchFieldsFromFieldTable(self, propId=None, degreesToRadians=True): 

419 """ 

420 Fetch field information (fieldID/RA/Dec) from the Field table. 

421 

422 This will select fields which were requested by a particular proposal or proposals, 

423 even if they did not receive any observations. 

424 

425 Parameters 

426 ---------- 

427 propId : int or list of ints 

428 Proposal ID or list of proposal IDs to use to select fields. 

429 degreesToRadians : bool, opt 

430 If True, convert degrees in Field table into radians. 

431 

432 Returns 

433 ------- 

434 np.recarray 

435 Structured array containing the field data (fieldID, fieldRA, fieldDec). 

436 """ 

437 if propId is not None: 

438 query = 'select f.fieldId, f.ra, f.dec from Field as f' 

439 query += ', ProposalField as p where (p.Field_fieldId = f.fieldId) ' 

440 if isinstance(propId, list) or isinstance(propId, np.ndarray): 

441 query += ' and (' 

442 for pID in propId: 

443 query += '(p.Proposal_propId = %d) or ' % (int(pID)) 

444 # Remove the trailing 'or' and add a closing parenthesis. 

445 query = query[:-3] 

446 query += ')' 

447 else: # single proposal ID. 

448 query += ' and (p.Proposal_propId = %d) ' %(int(propId)) 

449 query += ' group by f.%s' %(self.fieldIdCol) 

450 fielddata = self.query_arbitrary(query, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol], 

451 ['int', 'float', 'float']))) 

452 if len(fielddata) == 0: 

453 fielddata = np.zeros(0, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol], 

454 ['int', 'float', 'float']))) 

455 else: 

456 query = 'select fieldId, ra, dec from Field group by fieldId' 

457 fielddata = self.query_arbitrary(query, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol], 

458 ['int', 'float', 'float']))) 

459 if len(fielddata) == 0: 

460 fielddata = np.zeros(0, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol], 

461 ['int', 'float', 'float']))) 

462 if degreesToRadians: 

463 fielddata[self.raCol] = fielddata[self.raCol] * np.pi / 180. 

464 fielddata[self.decCol] = fielddata[self.decCol] * np.pi / 180. 

465 return fielddata 

466 

467 

468 def fetchPropInfo(self): 

469 """ 

470 Fetch the proposal IDs as well as their (short) proposal names and science type tags from the 

471 full opsim database. 

472 Returns dictionary of propID / propname, and dictionary of propTag / propID. 

473 If not using a full database, will return dict of propIDs with empty propnames + empty propTag dict. 

474 """ 

475 propIds = {} 

476 # Add WFD and DD tags by default to propTags as we expect these every time. (avoids key errors). 

477 propTags = {'WFD': [], 'DD': [], 'NES': []} 

478 assignData = True 

479 try: 

480 propData = self.query_columns('Proposal', colnames=[self.propIdCol, self.propNameCol], 

481 sqlconstraint=None) 

482 except ValueError: 

483 propData = [] 

484 propIds = {} 

485 propTags = {} 

486 assignData = False 

487 

488 if assignData: 

489 for propId, propName in zip(propData[self.propIdCol], propData[self.propNameCol]): 

490 # Fix these in the future, to use the proper tags that will be added to output database. 

491 propIds[propId] = propName 

492 if 'widefastdeep' in propName.lower(): 

493 propTags['WFD'].append(propId) 

494 if 'drilling' in propName.lower(): 

495 propTags['DD'].append(propId) 

496 if 'northeclipticspur' in propName.lower(): 

497 propTags['NES'].append(propId) 

498 return propIds, propTags 

499 

500 def createSlewConstraint(self, startTime=None, endTime=None): 

501 """Create a SQL constraint for the slew tables (slew activities, slew speeds, slew states) 

502 to select slews between startTime and endTime (MJD). 

503 

504 Parameters 

505 ---------- 

506 startTime : float or None, opt 

507 Time of first slew. Default (None) means no constraint on time of first slew. 

508 endTime : float or None, opt 

509 Time of last slew. Default (None) means no constraint on time of last slew. 

510 

511 Returns 

512 ------- 

513 str 

514 The SQL constraint, like 'slewHistory_slewID between XXX and XXXX' 

515 """ 

516 query = 'select min(observationStartMJD), max(observationStartMJD) from obsHistory' 

517 res = self.query_arbitrary(query, dtype=[('mjdMin', 'int'), ('mjdMax', 'int')]) 

518 # If asking for constraints that are out of bounds of the survey, return None. 

519 if startTime is None or (startTime < res['mjdMin'][0]): 

520 startTime = res['mjdMin'][0] 

521 if endTime is None or (endTime > res['mjdMax'][0]): 

522 endTime = res['mjdMax'][0] 

523 # Check if times were out of bounds. 

524 if startTime > res['mjdMax'][0] or endTime < res['mjdMin'][0]: 

525 warnings.warn('Times requested do not overlap survey operations (%f to %f)' 

526 % (res['mjdMin'][0], res['mjdMax'][0])) 

527 return None 

528 # Find the slew ID matching the start Time. 

529 query = 'select min(observationId) from obsHistory where observationStartMJD >= %s' % (startTime) 

530 res = self.query_arbitrary(query, dtype=[('observationId', 'int')]) 

531 obsHistId = res['observationId'][0] 

532 query = 'select slewCount from slewHistory where ObsHistory_observationId = %d' % (obsHistId) 

533 res = self.query_arbitrary(query, dtype=[('slewCount', 'int')]) 

534 minSlewCount = res['slewCount'][0] 

535 # Find the slew ID matching the end Time. 

536 query = 'select max(observationId) from obsHistory where observationStartMJD <= %s' % (endTime) 

537 res = self.query_arbitrary(query, dtype=[('observationId', 'int')]) 

538 obsHistId = res['observationId'][0] 

539 # Find the observation id corresponding to this slew time. 

540 query = 'select slewCount from slewHistory where ObsHistory_observationId = %d' % (obsHistId) 

541 res = self.query_arbitrary(query, dtype=[('slewCount', 'int')]) 

542 maxSlewCount = res['slewCount'][0] 

543 return 'SlewHistory_slewCount between %d and %d' % (minSlewCount, maxSlewCount) 

544 

545 def createSQLWhere(self, tag, propTags): 

546 """ 

547 Create a SQL constraint to identify observations taken for a particular proposal, 

548 using the information in the propTags dictionary. 

549 

550 Parameters 

551 ---------- 

552 tag : str 

553 The name of the proposal for which to create a SQLwhere clause (WFD or DD). 

554 propTags : dict 

555 A dictionary of {proposal name : [proposal ids]} 

556 This can be created using OpsimDatabase.fetchPropInfo() 

557 

558 Returns 

559 ------- 

560 str 

561 The SQL constraint, such as '(propID = 365) or (propID = 366)' 

562 """ 

563 if (tag not in propTags) or (len(propTags[tag]) == 0): 

564 print('No %s proposals found' % (tag)) 

565 # Create a sqlWhere clause that will not return anything as a query result. 

566 sqlWhere = 'proposalId like "NO PROP"' 

567 elif len(propTags[tag]) == 1: 

568 sqlWhere = "proposalId = %d" % (propTags[tag][0]) 

569 else: 

570 sqlWhere = "(" + " or ".join(["proposalId = %d" % (propid) for propid in propTags[tag]]) + ")" 

571 return sqlWhere 

572 

573 def fetchRequestedNvisits(self, propId=None): 

574 """Find the requested number of visits for the simulation or proposal(s). 

575 

576 Parameters 

577 ---------- 

578 propId : int or list of ints, opt 

579 

580 Returns 

581 ------- 

582 dict 

583 Number of visits in u/g/r/i/z/y. 

584 """ 

585 if propId is None: 

586 constraint = '' 

587 else: 

588 if isinstance(propId, int): 

589 constraint = 'propId = %d' % (propId) 

590 else: 

591 constraint = '' 

592 for pId in propId: 

593 constraint += 'propId = %d or ' % (int(pId)) 

594 constraint = constraint[:-3] 

595 propData = self.query_columns('Proposal', colnames=[self.propNameCol, self.propTypeCol], 

596 sqlconstraint=constraint) 

597 nvisits = {} 

598 for f in self.filterlist: 

599 nvisits[f] = 0 

600 for pName, propType in zip(propData[self.propNameCol], propData[self.propTypeCol]): 

601 if propType.lower() == 'general': 

602 for f in self.filterlist: 

603 constraint = 'paramName="science/general_props/values/%s/filters/%s/num_visits"' \ 

604 % (pName, f) 

605 val = self.query_columns('Config', colnames=['paramValue'], sqlconstraint=constraint) 

606 if len(val) > 0: 

607 nvisits[f] += int(val['paramValue'][0]) 

608 elif propType.lower == 'sequence': 

609 pass 

610 # Not clear yet. 

611 return nvisits 

612 

613 def _matchParamNameValue(self, configarray, keyword): 

614 return configarray['paramValue'][np.where(configarray['paramName'] == keyword)] 

615 

616 def _parseSequences(self, perPropConfig, filterlist): 

617 """ 

618 (Private). Given an array of config paramName/paramValue info for a given WLTSS proposal 

619 and the filterlist of filters used in that proposal, parse the sequences/subsequences and returns: 

620 a dictionary with all the per-sequence information (including subsequence names & events, etc.) 

621 a numpy array with the number of visits per filter 

622 """ 

623 propDict = {} 

624 # Identify where subsequences start in config[propname] arrays. 

625 seqidxs = np.where(perPropConfig['paramName'] == 'SubSeqName')[0] 

626 for sidx in seqidxs: 

627 i = sidx 

628 # Get the name of this subsequence. 

629 seqname = perPropConfig['paramValue'][i] 

630 # Check if seqname is a nested subseq of an existing sequence: 

631 nestedsubseq = False 

632 for prevseq in propDict: 

633 if 'SubSeqNested' in propDict[prevseq]: 

634 if seqname in propDict[prevseq]['SubSeqNested']: 

635 seqdict = propDict[prevseq]['SubSeqNested'][seqname] 

636 nestedsubseq = True 

637 # If not, then create a new subseqence key/dictionary for this subseq. 

638 if not nestedsubseq: 

639 propDict[seqname] = {} 

640 seqdict = propDict[seqname] 

641 # And move on to next parameters within subsequence set. 

642 i += 1 

643 if perPropConfig['paramName'][i] == 'SubSeqNested': 

644 subseqnestedname = perPropConfig['paramValue'][i] 

645 if subseqnestedname != '.': 

646 # Have nested subsequence, so keep track of that here 

647 # but will fill in info later. 

648 seqdict['SubSeqNested'] = {} 

649 # Set up nested dictionary for nested subsequence. 

650 seqdict['SubSeqNested'][subseqnestedname] = {} 

651 i += 1 

652 subseqfilters = perPropConfig['paramValue'][i] 

653 if subseqfilters != '.': 

654 seqdict['Filters'] = subseqfilters 

655 i += 1 

656 subseqexp = perPropConfig['paramValue'][i] 

657 if subseqexp != '.': 

658 seqdict['SubSeqExp'] = subseqexp 

659 i+= 1 

660 subseqevents = perPropConfig['paramValue'][i] 

661 seqdict['Events'] = int(subseqevents) 

662 i+=2 

663 subseqinterval = perPropConfig['paramValue'][i] 

664 subseqint = np.array([subseqinterval.split('*')], 'float').prod() 

665 # In days .. 

666 subseqint *= 1/24.0/60.0/60.0 

667 if subseqint > 1: 

668 seqdict['SubSeqInt'] = '%.2f days' %(subseqint) 

669 else: 

670 subseqint *= 24.0 

671 if subseqint > 1: 

672 seqdict['SubSeqInt'] = '%.2f hours' %(subseqint) 

673 else: 

674 subseqint *= 60.0 

675 seqdict['SubSeqInt'] = '%.3f minutes' %(subseqint) 

676 # End of assigning subsequence info - move on to counting number of visits. 

677 nvisits = np.zeros(len(filterlist), int) 

678 for subseq in propDict: 

679 subevents = propDict[subseq]['Events'] 

680 # Count visits from direct subsequences. 

681 if 'SubSeqExp' in propDict[subseq] and 'Filters' in propDict[subseq]: 

682 subfilters = propDict[subseq]['Filters'] 

683 subexp = propDict[subseq]['SubSeqExp'] 

684 # If just one filter .. 

685 if len(subfilters) == 1: 

686 idx = np.where(filterlist == subfilters)[0] 

687 nvisits[idx] += subevents * int(subexp) 

688 else: 

689 splitsubfilters = subfilters.split(',') 

690 splitsubexp = subexp.split(',') 

691 for f, exp in zip(splitsubfilters, splitsubexp): 

692 idx = np.where(filterlist == f)[0] 

693 nvisits[idx] += subevents * int(exp) 

694 # Count visits if have nested subsequences. 

695 if 'SubSeqNested' in propDict[subseq]: 

696 for subseqnested in propDict[subseq]['SubSeqNested']: 

697 events = subevents * propDict[subseq]['SubSeqNested'][subseqnested]['Events'] 

698 subfilters = propDict[subseq]['SubSeqNested'][subseqnested]['Filters'] 

699 subexp = propDict[subseq]['SubSeqNested'][subseqnested]['SubSeqExp'] 

700 # If just one filter .. 

701 if len(subfilters) == 1: 

702 idx = np.where(filterlist == subfilters)[0] 

703 nvisits[idx] += events * int(subexp) 

704 # Else may have multiple filters in the subsequence, so must split. 

705 splitsubfilters = subfilters.split(',') 

706 splitsubexp = subexp.split(',') 

707 for f, exp in zip(splitsubfilters, splitsubexp): 

708 idx = np.where(filterlist == f)[0] 

709 nvisits[idx] += int(exp) * events 

710 return propDict, nvisits 

711 

712 def _queryParam(self, constraint): 

713 results = self.query_columns('Config', colnames=['paramValue'], sqlconstraint=constraint) 

714 if len(results) > 0: 

715 return results['paramValue'][0] 

716 else: 

717 return '--' 

718 

719 def fetchConfig(self): 

720 """ 

721 Fetch config data from configTable, match proposal IDs with proposal names and some field data, 

722 and do a little manipulation of the data to make it easier to add to the presentation layer. 

723 """ 

724 # Check to see if we're dealing with a full database or not. If not, just return (no config info to fetch). 

725 if 'Session' not in self.tables: 

726 warnings.warn('Cannot fetch opsim config info as this is not a full opsim database.') 

727 return {}, {} 

728 # Create two dictionaries: a summary dict that contains a summary of the run 

729 configSummary = {} 

730 configSummary['keyorder'] = ['Version', 'RunInfo', 'Proposals'] 

731 # and the other a general dict that contains all the details (by group) of the run. 

732 config = {} 

733 # Start to build up the summary. 

734 # MAF version 

735 mafdate, mafversion = getDateVersion() 

736 configSummary['Version'] = {} 

737 configSummary['Version']['MAFVersion'] = '%s' %(mafversion['__version__']) 

738 configSummary['Version']['MAFDate'] = '%s' %(mafdate) 

739 # Opsim date, version and runcomment info from session table 

740 results = self.query_columns('Session', 

741 [self.versionCol, self.sessionDateCol, self.runCommentCol]) 

742 configSummary['Version']['OpsimVersion'] = '%s' %(results[self.versionCol][0]) 

743 configSummary['Version']['OpsimDate'] = '%s' %(results[self.sessionDateCol][0]) 

744 configSummary['Version']['OpsimDate'] = configSummary['Version']['OpsimDate'][0:10] 

745 configSummary['RunInfo'] = {} 

746 configSummary['RunInfo']['RunComment'] = results[self.runCommentCol][0] 

747 configSummary['RunInfo']['RunName'] = self.fetchOpsimRunName() 

748 # Pull out a few special values to put into summary. 

749 # This section has a number of configuration parameter names hard-coded. 

750 # I've left these here (rather than adding to self_colNames), bc I think schema changes in the config 

751 # files will actually be easier to track here (at least until the opsim configs are cleaned up). 

752 constraint = 'paramName="observatory/telescope/altitude_minpos"' 

753 configSummary['RunInfo']['MinAlt'] = self._queryParam(constraint) 

754 constraint = 'paramName="observatory/telescope/altitude_maxpos"' 

755 configSummary['RunInfo']['MaxAlt'] = self._queryParam(constraint) 

756 constraint = 'paramName="observatory/camera/filter_change_time"' 

757 configSummary['RunInfo']['TimeFilterChange'] = self._queryParam(constraint) 

758 constraint = 'paramName="observatory/camera/readout_time"' 

759 configSummary['RunInfo']['TimeReadout'] = self._queryParam(constraint) 

760 constraint = 'paramName="sched_driver/propboost_weight"' 

761 configSummary['RunInfo']['PropBoostWeight'] = self._queryParam(constraint) 

762 configSummary['RunInfo']['keyorder'] = ['RunName', 'RunComment', 'MinAlt', 'MaxAlt', 

763 'TimeFilterChange', 'TimeReadout', 'PropBoostWeight'] 

764 

765 # Echo config table into configDetails. 

766 configDetails = {} 

767 configs = self.query_columns('Config', ['paramName', 'paramValue']) 

768 for name, value in zip(configs['paramName'], configs['paramValue']): 

769 configDetails[name] = value 

770 

771 # Now finish building the summary to add proposal information. 

772 # Loop through all proposals to add summary information. 

773 propData = self.query_columns('Proposal', [self.propIdCol, self.propNameCol, self.propTypeCol]) 

774 configSummary['Proposals'] = {} 

775 for propid, propname, proptype in zip(propData[self.propIdCol], 

776 propData[self.propNameCol], propData[self.propTypeCol]): 

777 configSummary['Proposals'][propname] = {} 

778 propdict = configSummary['Proposals'][propname] 

779 propdict['keyorder'] = ['PropId', 'PropName', 'PropType', 'Airmass bonus', 'Airmass max', 

780 'HA bonus', 'HA max', 'Time weight', 'Restart Lost Sequences', 

781 'Restart Complete Sequences', 'Filters'] 

782 propdict['PropName'] = propname 

783 propdict['PropId'] = propid 

784 propdict['PropType'] = proptype 

785 # Add some useful information on the proposal parameters. 

786 constraint = 'paramName like "science/%s_props/values/%s/sky_constraints/max_airmass"'\ 

787 % ("%", propname) 

788 propdict['Airmass max'] = self._queryParam(constraint) 

789 constraint = 'paramName like "science/%s_props/values/%s/scheduling/airmass_bonus"'\ 

790 % ("%", propname) 

791 propdict['Airmass bonus'] = self._queryParam(constraint) 

792 constraint = 'paramName like "science/%s_props/values/%s/scheduling/hour_angle_max"'\ 

793 % ("%", propname) 

794 propdict['HA max'] = self._queryParam(constraint) 

795 constraint = 'paramName like "science/%s_props/values/%s/scheduling/hour_angle_bonus"'\ 

796 % ("%", propname) 

797 propdict['HA bonus'] = self._queryParam(constraint) 

798 constraint = 'paramName like "science/%s_props/values/%s/scheduling/time_weight"'\ 

799 % ("%", propname) 

800 propdict['Time weight'] = self._queryParam(constraint) 

801 constraint = 'paramName like "science/%s_props/values/%s/scheduling/restart_lost_sequences"'\ 

802 % ("%", propname) 

803 propdict['Restart Lost Sequences'] = self._queryParam(constraint) 

804 constraint = 'paramName like "science/%s_props/values/%s/scheduling/restart_complete_sequences"'\ 

805 % ("%", propname) 

806 propdict['Restart Complete Sequences'] = self._queryParam(constraint) 

807 # Find number of visits requested per filter for the proposal 

808 # along with min/max sky and airmass values. 

809 propdict['Filters'] = {} 

810 for f in self.filterlist: 

811 propdict['Filters'][f] = {} 

812 propdict['Filters'][f]['Filter'] = f 

813 dictkeys = ['MaxSeeing', 'BrightLimit', 'DarkLimit', 'NumVisits', 'GroupedVisits', 'Snaps'] 

814 querykeys = ['max_seeing', 'bright_limit', 'dark_limit', 'num_visits', 

815 'num_grouped_visits', 'exposures'] 

816 for dk, qk in zip(dictkeys, querykeys): 

817 constraint = 'paramName like "science/%s_props/values/%s/filters/%s/%s"' \ 

818 % ("%", propname, f, qk) 

819 propdict['Filters'][f][dk] = self._queryParam(constraint) 

820 propdict['Filters'][f]['keyorder'] = ['Filter', 'MaxSeeing', 'MinSky', 'MaxSky', 

821 'NumVisits', 'GroupedVisits', 'Snaps'] 

822 propdict['Filters']['keyorder'] = list(self.filterlist) 

823 return configSummary, configDetails 

824 

825 

826class OpsimDatabaseV3(BaseOpsimDatabase): 

827 def __init__(self, database, driver='sqlite', host=None, port=None, defaultTable='Summary', 

828 longstrings=False, verbose=False): 

829 """ 

830 Instantiate object to handle queries of the opsim database. 

831 (In general these will be the sqlite database files produced by opsim, but could 

832 be any database holding those opsim output tables.). 

833 

834 database = Name of database or sqlite filename 

835 driver = Name of database dialect+driver for sqlalchemy (e.g. 'sqlite', 'pymssql+mssql') 

836 host = Name of database host (optional) 

837 port = String port number (optional) 

838 

839 """ 

840 super(OpsimDatabaseV3, self).__init__(database=database, driver=driver, host=host, port=port, 

841 defaultTable=defaultTable, longstrings=longstrings, 

842 verbose=verbose) 

843 

844 def _colNames(self): 

845 """ 

846 Set variables to represent the common column names used in this class directly. 

847 

848 This should make future schema changes a little easier to handle. 

849 It is NOT meant to function as a general column map, just to abstract values 

850 which are used *within this class*. 

851 """ 

852 self.mjdCol = 'expMJD' 

853 self.slewId = 'slewHistory_slewID' 

854 self.delayCol = 'actDelay' 

855 self.fieldIdCol = 'fieldID' 

856 self.raCol = 'fieldRA' 

857 self.decCol = 'fieldDec' 

858 self.propIdCol = 'propID' 

859 self.propConfCol = 'propConf' 

860 self.propNameCol = 'propName' #(propname == proptype) 

861 # For config parsing. 

862 self.versionCol = 'version' 

863 self.sessionIdCol = 'sessionID' 

864 self.sessionHostCol = 'sessionHost' 

865 self.sessionDateCol = 'sessionDate' 

866 self.runCommentCol = 'runComment' 

867 self.runLengthParam = 'nRun' 

868 self.raDecInDeg = False 

869 self.opsimVersion = 'V3' 

870 

871 def fetchFieldsFromFieldTable(self, propId=None, degreesToRadians=True): 

872 """ 

873 Fetch field information (fieldID/RA/Dec) from Field (+Proposal_Field) tables. 

874 

875 propID = the proposal ID (default None), if selecting particular proposal - can be a list 

876 degreesToRadians = RA/Dec values are in degrees in the Field table (so convert to radians). 

877 """ 

878 # Note that you can't select any other sql constraints (such as filter). 

879 # This will select fields which were requested by a particular proposal or proposals, 

880 # even if they didn't get any observations. 

881 if propId is not None: 

882 query = 'select f.%s, f.%s, f.%s from %s as f' %(self.fieldIdCol, self.raCol, self.decCol, 

883 'Field') 

884 query += ', %s as p where (p.Field_%s = f.%s) ' %('Proposal_Field', 

885 self.fieldIdCol, self.fieldIdCol) 

886 if isinstance(propId, list) or isinstance(propId, np.ndarray): 

887 query += ' and (' 

888 for pID in propId: 

889 query += '(p.Proposal_%s = %d) or ' %(self.propIdCol, int(pID)) 

890 # Remove the trailing 'or' and add a closing parenthesis. 

891 query = query[:-3] 

892 query += ')' 

893 else: # single proposal ID. 

894 query += ' and (p.Proposal_%s = %d) ' %(self.propIdCol, int(propId)) 

895 query += ' group by f.%s' %(self.fieldIdCol) 

896 fielddata = self.query_arbitrary(query, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol], 

897 ['int', 'float', 'float']))) 

898 if len(fielddata) == 0: 

899 fielddata = np.zeros(0, dtype=list(zip([self.fieldIdCol, self.raCol, self.decCol], 

900 ['int', 'float', 'float']))) 

901 else: 

902 fielddata = self.query_columns('Field', colnames=[self.fieldIdCol, self.raCol, self.decCol], 

903 groupBy = self.fieldIdCol) 

904 if degreesToRadians: 

905 fielddata[self.raCol] = fielddata[self.raCol] * np.pi / 180. 

906 fielddata[self.decCol] = fielddata[self.decCol] * np.pi / 180. 

907 return fielddata 

908 

909 def fetchPropInfo(self): 

910 """ 

911 Fetch the proposal IDs as well as their (short) proposal names and science type tags from the 

912 full opsim database. 

913 Returns dictionary of propID / propname, and dictionary of propTag / propID. 

914 If not using a full database, will return dict of propIDs with empty propnames + empty propTag dict. 

915 """ 

916 propIDs = {} 

917 # Add WFD and DD tags by default to propTags as we expect these every time. (avoids key errors). 

918 propTags = {'WFD':[], 'DD':[], 'NES': []} 

919 # If do not have full database available: 

920 if 'Proposal' not in self.tables: 

921 propData = self.query_columns(self.defaultTable, colnames=[self.propIdCol]) 

922 for propid in propData[self.propIdCol]: 

923 propIDs[int(propid)] = propid 

924 else: 

925 # Query for all propIDs. 

926 propData = self.query_columns('Proposal', colnames=[self.propIdCol, self.propConfCol, 

927 self.propNameCol], sqlconstraint=None) 

928 for propid, propname in zip(propData[self.propIdCol], propData[self.propConfCol]): 

929 # Strip '.conf', 'Prop', and path info. 

930 propIDs[int(propid)] = re.sub('Prop','', re.sub('.conf','', re.sub('.*/', '', propname))) 

931 # Find the 'ScienceType' from the config table, to indicate DD/WFD/Rolling, etc. 

932 sciencetypes = self.query_columns('Config', colnames=['paramValue', 'nonPropID'], 

933 sqlconstraint="paramName like 'ScienceType'") 

934 if len(sciencetypes) == 0: 

935 # Then this was an older opsim run without 'ScienceType' tags, 

936 # so fall back to trying to guess what proposals are WFD or DD. 

937 for propid, propname in propIDs.items(): 

938 if 'universal' in propname.lower(): 

939 propTags['WFD'].append(propid) 

940 if 'deep' in propname.lower(): 

941 propTags['DD'].append(propid) 

942 if 'northecliptic' in propname.lower(): 

943 propTags['NES'].append(propid) 

944 else: 

945 # Newer opsim output with 'ScienceType' fields in conf files. 

946 for sc in sciencetypes: 

947 # ScienceType tag can be multiple values, separated by a ',' 

948 tags = [x.strip(' ') for x in sc['paramValue'].split(',')] 

949 for sciencetype in tags: 

950 if sciencetype in propTags: 

951 propTags[sciencetype].append(int(sc['nonPropID'])) 

952 else: 

953 propTags[sciencetype] = [int(sc['nonPropID']),] 

954 # But even these runs don't tag NES 

955 for propid, propname in propIDs.items(): 

956 if 'northecliptic' in propname.lower(): 

957 propTags['NES'].append(propid) 

958 return propIDs, propTags 

959 

960 def createSlewConstraint(self, startTime=None, endTime=None): 

961 """Create a SQL constraint for the slew tables (slew activities, slew speeds, slew states) 

962 to select slews between startTime and endTime (MJD). 

963 

964 Parameters 

965 ---------- 

966 startTime : float or None, opt 

967 Time of first slew. Default (None) means no constraint on time of first slew. 

968 endTime : float or None, opt 

969 Time of last slew. Default (None) means no constraint on time of last slew. 

970 

971 Returns 

972 ------- 

973 str 

974 The SQL constraint, like 'slewHistory_slewID > XXX and slewHistory_slewID < XXXX' 

975 """ 

976 query = 'select min(expMJD), max(expMJD) from obsHistory' 

977 res = self.query_arbitrary(query, dtype=[('mjdMin', 'int'), ('mjdMax', 'int')]) 

978 # If asking for constraints that are out of bounds of the survey, return None. 

979 if startTime is None or (startTime < res['mjdMin'][0]): 

980 startTime = res['mjdMin'][0] 

981 if endTime is None or (endTime > res['mjdMax'][0]): 

982 endTime = res['mjdMax'][0] 

983 # Check if times were out of bounds. 

984 if startTime > res['mjdMax'][0] or endTime < res['mjdMin'][0]: 

985 warnings.warn('Times requested do not overlap survey operations (%f to %f)' 

986 % (res['mjdMin'][0], res['mjdMax'][0])) 

987 return None 

988 # Find the slew ID matching the start Time. 

989 query = 'select min(obsHistID) from obsHistory where expMJD >= %s' % (startTime) 

990 res = self.query_arbitrary(query, dtype=[('observationId', 'int')]) 

991 obsHistId = res['observationId'][0] 

992 query = 'select slewID from slewHistory where ObsHistory_obsHistID = %d' % (obsHistId) 

993 res = self.query_arbitrary(query, dtype=[('slewCount', 'int')]) 

994 minSlewCount = res['slewCount'][0] 

995 # Find the slew ID matching the end Time. 

996 query = 'select max(obsHistID) from obsHistory where expMJD <= %s' % (endTime) 

997 res = self.query_arbitrary(query, dtype=[('observationId', 'int')]) 

998 obsHistId = res['observationId'][0] 

999 # Find the observation id corresponding to this slew time. 

1000 query = 'select slewID from slewHistory where ObsHistory_obsHistID = %d' % (obsHistId) 

1001 res = self.query_arbitrary(query, dtype=[('slewCount', 'int')]) 

1002 maxSlewCount = res['slewCount'][0] 

1003 return 'SlewHistory_slewID between %d and %d' % (minSlewCount, maxSlewCount) 

1004 

1005 def createSQLWhere(self, tag, propTags): 

1006 """Create a SQL constraint to identify observations taken for a particular proposal, 

1007 using the information in the propTags dictionary. 

1008 

1009 Parameters 

1010 ---------- 

1011 tag : str 

1012 The name of the proposal for which to create a SQLwhere clause (WFD or DD). 

1013 propTags : dict 

1014 A dictionary of {proposal name : [proposal ids]} 

1015 This can be created using OpsimDatabase.fetchPropInfo() 

1016 

1017 Returns 

1018 ------- 

1019 str 

1020 The SQL constraint, such as '(propID = 365) or (propID = 366)' 

1021 """ 

1022 if (tag not in propTags) or (len(propTags[tag]) == 0): 

1023 print('No %s proposals found' % (tag)) 

1024 # Create a sqlWhere clause that will not return anything as a query result. 

1025 sqlWhere = 'propID like "NO PROP"' 

1026 elif len(propTags[tag]) == 1: 

1027 sqlWhere = "propID = %d" % (propTags[tag][0]) 

1028 else: 

1029 sqlWhere = "(" + " or ".join(["propID = %d" % (propid) for propid in propTags[tag]]) + ")" 

1030 return sqlWhere 

1031 

1032 def fetchRequestedNvisits(self, propId=None): 

1033 """ 

1034 Find the requested number of visits for proposals in propId. 

1035 Returns a dictionary - Nvisits{u/g/r/i/z/y} 

1036 """ 

1037 visitDict = {} 

1038 if propId is None: 

1039 # Get all the available propIds. 

1040 propData = self.query_columns('Proposal', colnames=[self.propIdCol, self.propNameCol], 

1041 sqlconstraint=None) 

1042 else: 

1043 # Get the propType info to go with the propId(s). 

1044 if hasattr(propId, '__iter__'): 

1045 constraint = '(' 

1046 for pi in propId: 

1047 constraint += '(propId = %d) or ' %(pi) 

1048 constraint = constraint[:-4] + ')' 

1049 else: 

1050 constraint = 'propId = %d' %(propId) 

1051 propData = self.query_columns('Proposal', colnames=[self.propIdCol, self.propNameCol], 

1052 sqlconstraint=constraint) 

1053 for pId, propType in zip(propData[self.propIdCol], propData[self.propNameCol]): 

1054 perPropConfig = self.query_columns('Config', colnames=['paramName', 'paramValue'], 

1055 sqlconstraint = 'nonPropID = %d and paramName!="userRegion"' 

1056 %(pId)) 

1057 filterlist = self._matchParamNameValue(perPropConfig, 'Filter') 

1058 if propType == 'WL': 

1059 # For WL proposals, the simple 'Filter_Visits' == the requested number of observations. 

1060 nvisits = np.array(self._matchParamNameValue(perPropConfig, 'Filter_Visits'), int) 

1061 elif propType == 'WLTSS': 

1062 seqDict, nvisits = self._parseSequences(perPropConfig, filterlist) 

1063 visitDict[pId] = {} 

1064 for f, N in zip(filterlist, nvisits): 

1065 visitDict[pId][f] = N 

1066 nvisits = {} 

1067 for f in ['u', 'g', 'r', 'i', 'z', 'y']: 

1068 nvisits[f] = 0 

1069 for pId in visitDict: 

1070 for f in visitDict[pId]: 

1071 nvisits[f] += visitDict[pId][f] 

1072 return nvisits 

1073 

1074 def _matchParamNameValue(self, configarray, keyword): 

1075 return configarray['paramValue'][np.where(configarray['paramName']==keyword)] 

1076 

1077 def _parseSequences(self, perPropConfig, filterlist): 

1078 """ 

1079 (Private). Given an array of config paramName/paramValue info for a given WLTSS proposal 

1080 and the filterlist of filters used in that proposal, parse the sequences/subsequences and returns: 

1081 a dictionary with all the per-sequence information (including subsequence names & events, etc.) 

1082 a numpy array with the number of visits per filter 

1083 """ 

1084 propDict = {} 

1085 # Identify where subsequences start in config[propname] arrays. 

1086 seqidxs = np.where(perPropConfig['paramName'] == 'SubSeqName')[0] 

1087 for sidx in seqidxs: 

1088 i = sidx 

1089 # Get the name of this subsequence. 

1090 seqname = perPropConfig['paramValue'][i] 

1091 # Check if seqname is a nested subseq of an existing sequence: 

1092 nestedsubseq = False 

1093 for prevseq in propDict: 

1094 if 'SubSeqNested' in propDict[prevseq]: 

1095 if seqname in propDict[prevseq]['SubSeqNested']: 

1096 seqdict = propDict[prevseq]['SubSeqNested'][seqname] 

1097 nestedsubseq = True 

1098 # If not, then create a new subseqence key/dictionary for this subseq. 

1099 if not nestedsubseq: 

1100 propDict[seqname] = {} 

1101 seqdict = propDict[seqname] 

1102 # And move on to next parameters within subsequence set. 

1103 i += 1 

1104 if perPropConfig['paramName'][i] == 'SubSeqNested': 

1105 subseqnestedname = perPropConfig['paramValue'][i] 

1106 if subseqnestedname != '.': 

1107 # Have nested subsequence, so keep track of that here 

1108 # but will fill in info later. 

1109 seqdict['SubSeqNested'] = {} 

1110 # Set up nested dictionary for nested subsequence. 

1111 seqdict['SubSeqNested'][subseqnestedname] = {} 

1112 i += 1 

1113 subseqfilters = perPropConfig['paramValue'][i] 

1114 if subseqfilters != '.': 

1115 seqdict['Filters'] = subseqfilters 

1116 i += 1 

1117 subseqexp = perPropConfig['paramValue'][i] 

1118 if subseqexp != '.': 

1119 seqdict['SubSeqExp'] = subseqexp 

1120 i+= 1 

1121 subseqevents = perPropConfig['paramValue'][i] 

1122 seqdict['Events'] = int(subseqevents) 

1123 i+=2 

1124 subseqinterval = perPropConfig['paramValue'][i] 

1125 subseqint = np.array([subseqinterval.split('*')], 'float').prod() 

1126 # In days .. 

1127 subseqint *= 1/24.0/60.0/60.0 

1128 if subseqint > 1: 

1129 seqdict['SubSeqInt'] = '%.2f days' %(subseqint) 

1130 else: 

1131 subseqint *= 24.0 

1132 if subseqint > 1: 

1133 seqdict['SubSeqInt'] = '%.2f hours' %(subseqint) 

1134 else: 

1135 subseqint *= 60.0 

1136 seqdict['SubSeqInt'] = '%.3f minutes' %(subseqint) 

1137 # End of assigning subsequence info - move on to counting number of visits. 

1138 nvisits = np.zeros(len(filterlist), int) 

1139 for subseq in propDict: 

1140 subevents = propDict[subseq]['Events'] 

1141 # Count visits from direct subsequences. 

1142 if 'SubSeqExp' in propDict[subseq] and 'Filters' in propDict[subseq]: 

1143 subfilters = propDict[subseq]['Filters'] 

1144 subexp = propDict[subseq]['SubSeqExp'] 

1145 # If just one filter .. 

1146 if len(subfilters) == 1: 

1147 idx = np.where(filterlist == subfilters)[0] 

1148 nvisits[idx] += subevents * int(subexp) 

1149 else: 

1150 splitsubfilters = subfilters.split(',') 

1151 splitsubexp = subexp.split(',') 

1152 for f, exp in zip(splitsubfilters, splitsubexp): 

1153 idx = np.where(filterlist == f)[0] 

1154 nvisits[idx] += subevents * int(exp) 

1155 # Count visits if have nested subsequences. 

1156 if 'SubSeqNested' in propDict[subseq]: 

1157 for subseqnested in propDict[subseq]['SubSeqNested']: 

1158 events = subevents * propDict[subseq]['SubSeqNested'][subseqnested]['Events'] 

1159 subfilters = propDict[subseq]['SubSeqNested'][subseqnested]['Filters'] 

1160 subexp = propDict[subseq]['SubSeqNested'][subseqnested]['SubSeqExp'] 

1161 # If just one filter .. 

1162 if len(subfilters) == 1: 

1163 idx = np.where(filterlist == subfilters)[0] 

1164 nvisits[idx] += events * int(subexp) 

1165 # Else may have multiple filters in the subsequence, so must split. 

1166 splitsubfilters = subfilters.split(',') 

1167 splitsubexp = subexp.split(',') 

1168 for f, exp in zip(splitsubfilters, splitsubexp): 

1169 idx = np.where(filterlist == f)[0] 

1170 nvisits[idx] += int(exp) * events 

1171 return propDict, nvisits 

1172 

1173 def fetchConfig(self): 

1174 """ 

1175 Fetch config data from configTable, match proposal IDs with proposal names and some field data, 

1176 and do a little manipulation of the data to make it easier to add to the presentation layer. 

1177 """ 

1178 # Check to see if we're dealing with a full database or not. If not, just return (no config info to fetch). 

1179 if 'Session' not in self.tables: 

1180 warnings.warn('Cannot fetch opsim config info as this is not a full opsim database.') 

1181 return {}, {} 

1182 # Create two dictionaries: a summary dict that contains a summary of the run 

1183 configSummary = {} 

1184 configSummary['keyorder'] = ['Version', 'RunInfo', 'Proposals'] 

1185 # and the other a general dict that contains all the details (by group) of the run. 

1186 config = {} 

1187 # Start to build up the summary. 

1188 # MAF version 

1189 mafdate, mafversion = getDateVersion() 

1190 configSummary['Version'] = {} 

1191 configSummary['Version']['MAFVersion'] = '%s' % (mafversion['__version__']) 

1192 configSummary['Version']['MAFDate'] = '%s' % (mafdate) 

1193 # Opsim date, version and runcomment info from session table 

1194 results = self.query_columns('Session', colnames = [self.versionCol, self.sessionDateCol, 

1195 self.runCommentCol]) 

1196 configSummary['Version']['OpsimVersion'] = '%s' % (results[self.versionCol][0]) 

1197 configSummary['Version']['OpsimDate'] = '%s' % (results[self.sessionDateCol][0]) 

1198 configSummary['RunInfo'] = {} 

1199 configSummary['RunInfo']['RunComment'] = results[self.runCommentCol] 

1200 configSummary['RunInfo']['RunName'] = self.fetchOpsimRunName() 

1201 # Pull out a few special values to put into summary. 

1202 # This section has a number of configuration parameter names hard-coded. 

1203 # I've left these here (rather than adding to self_colNames), because I think schema changes will in the config 

1204 # files will actually be easier to track here (at least until the opsim configs are cleaned up). 

1205 constraint = 'moduleName="Config" and paramName="sha1"' 

1206 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint) 

1207 try: 

1208 configSummary['Version']['Config sha1'] = results['paramValue'][0] 

1209 except IndexError: 

1210 configSummary['Version']['Config sha1'] = 'Unavailable' 

1211 constraint = 'moduleName="Config" and paramName="changedFiles"' 

1212 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint) 

1213 try: 

1214 configSummary['Version']['Config changed files'] = results['paramValue'][0] 

1215 except IndexError: 

1216 configSummary['Version']['Config changed files'] = 'Unavailable' 

1217 constraint = 'moduleName="instrument" and paramName="Telescope_AltMin"' 

1218 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint) 

1219 configSummary['RunInfo']['MinAlt'] = results['paramValue'][0] 

1220 constraint = 'moduleName="instrument" and paramName="Telescope_AltMax"' 

1221 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint) 

1222 configSummary['RunInfo']['MaxAlt'] = results['paramValue'][0] 

1223 constraint = 'moduleName="instrument" and paramName="Filter_MoveTime"' 

1224 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint) 

1225 configSummary['RunInfo']['TimeFilterChange'] = results['paramValue'][0] 

1226 constraint = 'moduleName="instrument" and paramName="Readout_Time"' 

1227 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint) 

1228 configSummary['RunInfo']['TimeReadout'] = results['paramValue'][0] 

1229 constraint = 'moduleName="scheduler" and paramName="MinDistance2Moon"' 

1230 results = self.query_columns('Config', colnames=['paramValue', ], sqlconstraint=constraint) 

1231 configSummary['RunInfo']['MinDist2Moon'] = results['paramValue'][0] 

1232 configSummary['RunInfo']['keyorder'] = ['RunName', 'RunComment', 'MinDist2Moon', 'MinAlt', 'MaxAlt', 

1233 'TimeFilterChange', 'TimeReadout'] 

1234 # Now build up config dict with 'nice' group names (proposal name and short module name) 

1235 # Each dict entry is a numpy array with the paramName/paramValue/comment values. 

1236 # Match proposal IDs with names. 

1237 query = 'select %s, %s, %s from Proposal group by %s' %(self.propIdCol, self.propConfCol, 

1238 self.propNameCol, self.propIdCol) 

1239 propdata = self.query_arbitrary(query, dtype=([(self.propIdCol, int), 

1240 (self.propConfCol, np.str, 256), 

1241 (self.propNameCol, np.str, 256)])) 

1242 # Make 'nice' proposal names 

1243 propnames = np.array([os.path.split(x)[1].replace('.conf', '') for x in propdata[self.propConfCol]]) 

1244 # Get 'nice' module names 

1245 moduledata = self.query_columns('Config', colnames=['moduleName',], sqlconstraint='nonPropID=0') 

1246 modulenames = np.array([os.path.split(x)[1].replace('.conf', '') for x in moduledata['moduleName']]) 

1247 # Grab the config information for each proposal and module. 

1248 cols = ['paramName', 'paramValue', 'comment'] 

1249 for longmodname, modname in zip(moduledata['moduleName'], modulenames): 

1250 config[modname] = self.query_columns('Config', colnames=cols, 

1251 sqlconstraint='moduleName="%s"' %(longmodname)) 

1252 config[modname] = config[modname][['paramName', 'paramValue', 'comment']] 

1253 for propid, propname in zip(propdata[self.propIdCol], propnames): 

1254 config[propname] = self.query_columns('Config', colnames=cols, 

1255 sqlconstraint='nonPropID="%s" and paramName!="userRegion"' 

1256 %(propid)) 

1257 config[propname] = config[propname][['paramName', 'paramValue', 'comment']] 

1258 config['keyorder'] = ['Comment', 'LSST', 'site', 'instrument', 'filters', 

1259 'AstronomicalSky', 'File', 'scheduler', 

1260 'schedulingData', 'schedDown', 'unschedDown'] 

1261 # Now finish building the summary to add proposal information. 

1262 # Loop through all proposals to add summary information. 

1263 configSummary['Proposals'] = {} 

1264 propidorder = sorted(propdata[self.propIdCol]) 

1265 # Generate a keyorder to print proposals in order of propid. 

1266 configSummary['Proposals']['keyorder'] = [] 

1267 for propid in propidorder: 

1268 configSummary['Proposals']['keyorder'].append(propnames[np.where(propdata[self.propIdCol] 

1269 == propid)][0]) 

1270 for propid, propname in zip(propdata[self.propIdCol], propnames): 

1271 configSummary['Proposals'][propname] = {} 

1272 propdict = configSummary['Proposals'][propname] 

1273 propdict['keyorder'] = [self.propIdCol, self.propNameCol, 'PropType', 'RelPriority', 

1274 'NumUserRegions', 'NumFields'] 

1275 propdict[self.propNameCol] = propname 

1276 propdict[self.propIdCol] = propid 

1277 propdict['PropType'] = propdata[self.propNameCol][np.where(propnames == propname)] 

1278 propdict['RelPriority'] = self._matchParamNameValue(config[propname], 'RelativeProposalPriority') 

1279 # Get the number of user regions. 

1280 constraint = 'nonPropID="%s" and paramName="userRegion"' %(propid) 

1281 result = self.query_columns('Config', colnames=['paramName',], sqlconstraint=constraint) 

1282 propdict['NumUserRegions'] = result.size 

1283 # Get the number of fields requested in the proposal (all filters). 

1284 propdict['NumFields'] = self.fetchFieldsFromFieldTable(propId=propid).size 

1285 # Find number of visits requested per filter for the proposal, with min/max sky & airmass values. 

1286 # Note that config table has multiple entries for Filter/Filter_Visits/etc. with the same name. 

1287 # The order of these entries in the config array matters. 

1288 propdict['PerFilter'] = {} 

1289 for key, keyword in zip(['Filters', 'MaxSeeing', 'MinSky', 'MaxSky'], 

1290 ['Filter', 'Filter_MaxSeeing', 'Filter_MinBrig', 'Filter_MaxBrig']): 

1291 temp = self._matchParamNameValue(config[propname], keyword) 

1292 if len(temp) > 0: 

1293 propdict['PerFilter'][key] = temp 

1294 # Add exposure time, potentially looking for scaling per filter. 

1295 exptime = float(self._matchParamNameValue(config[propname], 'ExposureTime')[0]) 

1296 temp = self._matchParamNameValue(config[propname], 'Filter_ExpFactor') 

1297 if len(temp) > 0: 

1298 propdict['PerFilter']['VisitTime'] = temp * exptime 

1299 else: 

1300 propdict['PerFilter']['VisitTime'] = np.ones(len(propdict['PerFilter']['Filters']), float) 

1301 propdict['PerFilter']['VisitTime'] *= exptime 

1302 # And count how many total exposures are requested per filter. 

1303 # First check if 'RestartCompleteSequences' is true: 

1304 # if both are true, then basically an indefinite number of visits are requested, although we're 

1305 # not going to count this way (as the proposals still make an approximate number of requests). 

1306 restartComplete = False 

1307 temp = self._matchParamNameValue(config[propname], 'RestartCompleteSequences') 

1308 if len(temp) > 0: 

1309 if temp[0] == 'True': 

1310 restartComplete = True 

1311 propdict['RestartCompleteSequences'] = restartComplete 

1312 # Grab information on restarting lost sequences so we can print this too. 

1313 restartLost = False 

1314 tmp = self._matchParamNameValue(config[propname], 'RestartLostSequences') 

1315 if len(temp) > 0: 

1316 if temp[0] == 'True': 

1317 restartLost = True 

1318 propdict['RestartLostSequences'] = restartLost 

1319 if propdict['PropType'] == 'WL': 

1320 # Simple 'Filter_Visits' request for number of observations. 

1321 propdict['PerFilter']['NumVisits'] = np.array(self._matchParamNameValue(config[propname], 

1322 'Filter_Visits'), int) 

1323 elif propdict['PropType'] == 'WLTSS': 

1324 # Proposal contains subsequences and possible nested subseq, so must delve further. 

1325 # Make a dictionary to hold the subsequence info (keyed per subsequence). 

1326 propdict['SubSeq'], Nvisits = self._parseSequences(config[propname], 

1327 propdict['PerFilter']['Filters']) 

1328 propdict['PerFilter']['NumVisits'] = Nvisits 

1329 propdict['SubSeq']['keyorder'] = ['SubSeqName', 'SubSeqNested', 'Events', 'SubSeqInt'] 

1330 # Sort the filter information so it's ugrizy instead of order in opsim config db. 

1331 idx = [] 

1332 for f in self.filterlist: 

1333 filterpos = np.where(propdict['PerFilter']['Filters'] == f) 

1334 if len(filterpos[0]) > 0: 

1335 idx.append(filterpos[0][0]) 

1336 idx = np.array(idx, int) 

1337 for k in propdict['PerFilter']: 

1338 tmp = propdict['PerFilter'][k][idx] 

1339 propdict['PerFilter'][k] = tmp 

1340 propdict['PerFilter']['keyorder'] = ['Filters', 'VisitTime', 'MaxSeeing', 'MinSky', 

1341 'MaxSky', 'NumVisits'] 

1342 return configSummary, config 

1343