Coverage for python/lsst/summit/utils/efdUtils.py: 15%

137 statements  

« prev     ^ index     » next       coverage.py v7.3.0, created at 2023-08-29 10:24 +0000

1# This file is part of summit_utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import asyncio 

23from astropy.time import Time, TimeDelta 

24from astropy import units as u 

25import datetime 

26import logging 

27import pandas as pd 

28 

29from .utils import getSite 

30 

31HAS_EFD_CLIENT = True 

32try: 

33 from lsst_efd_client import EfdClient 

34except ImportError: 

35 HAS_EFD_CLIENT = False 

36 

37__all__ = [ 

38 'getEfdData', 

39 'getMostRecentRowWithDataBefore', 

40 'makeEfdClient', 

41 'expRecordToTimespan', 

42 'efdTimestampToAstropy', 

43 'astropyToEfdTimestamp', 

44 'clipDataToEvent', 

45 'calcNextDay', 

46 'getDayObsStartTime', 

47 'getDayObsEndTime', 

48 'getDayObsForTime', 

49 'getSubTopics', 

50] 

51 

52 

53COMMAND_ALIASES = { 

54 'raDecTarget': 'lsst.sal.MTPtg.command_raDecTarget', 

55 'moveToTarget': 'lsst.sal.MTMount.command_moveToTarget', 

56 'startTracking': 'lsst.sal.MTMount.command_startTracking', 

57 'stopTracking': 'lsst.sal.MTMount.command_stopTracking', 

58 'trackTarget': 'lsst.sal.MTMount.command_trackTarget', # issued at 20Hz - don't plot 

59} 

60 

61# When looking backwards in time to find the most recent state event, look back 

62# in chunks of this size. Too small, and there will be too many queries, too 

63# large and there will be too much data returned unnecessarily, as we only need 

64# one row by definition. Will tune this parameters in consultation with SQuaRE. 

65TIME_CHUNKING = datetime.timedelta(minutes=15) 

66 

67 

68def _getBeginEnd(dayObs=None, begin=None, end=None, timespan=None, event=None, expRecord=None): 

69 """Calculate the begin and end times to pass to _getEfdData, given the 

70 kwargs passed to getEfdData. 

71 

72 Parameters 

73 ---------- 

74 dayObs : `int` 

75 The dayObs to query. If specified, this is used to determine the begin 

76 and end times. 

77 begin : `astropy.Time` 

78 The begin time for the query. If specified, either an end time or a 

79 timespan must be supplied. 

80 end : `astropy.Time` 

81 The end time for the query. If specified, a begin time must also be 

82 supplied. 

83 timespan : `astropy.TimeDelta` 

84 The timespan for the query. If specified, a begin time must also be 

85 supplied. 

86 event : `lsst.summit.utils.efdUtils.TmaEvent` 

87 The event to query. If specified, this is used to determine the begin 

88 and end times, and all other options are disallowed. 

89 expRecord : `lsst.daf.butler.dimensions.DimensionRecord` 

90 The exposure record containing the timespan to query. If specified, all 

91 other options are disallowed. 

92 

93 Returns 

94 ------- 

95 begin : `astropy.Time` 

96 The begin time for the query. 

97 end : `astropy.Time` 

98 The end time for the query. 

99 """ 

100 if expRecord is not None: 

101 forbiddenOpts = [event, begin, end, timespan, dayObs] 

102 if any(x is not None for x in forbiddenOpts): 

103 raise ValueError("You can't specify both an expRecord and a begin/end or timespan or dayObs") 

104 begin = expRecord.timespan.begin 

105 end = expRecord.timespan.end 

106 return begin, end 

107 

108 if event is not None: 

109 forbiddenOpts = [begin, end, timespan, dayObs] 

110 if any(x is not None for x in forbiddenOpts): 

111 raise ValueError("You can't specify both an event and a begin/end or timespan or dayObs") 

112 begin = event.begin 

113 end = event.end 

114 return begin, end 

115 

116 # check for dayObs, and that other options aren't inconsistently specified 

117 if dayObs is not None: 

118 forbiddenOpts = [begin, end, timespan] 

119 if any(x is not None for x in forbiddenOpts): 

120 raise ValueError("You can't specify both a dayObs and a begin/end or timespan") 

121 begin = getDayObsStartTime(dayObs) 

122 end = getDayObsEndTime(dayObs) 

123 return begin, end 

124 # can now disregard dayObs entirely 

125 

126 if begin is None: 

127 raise ValueError("You must specify either a dayObs or a begin/end or begin/timespan") 

128 # can now rely on begin, so just need to deal with end/timespan 

129 

130 if end is None and timespan is None: 

131 raise ValueError("If you specify a begin, you must specify either a end or a timespan") 

132 if end is not None and timespan is not None: 

133 raise ValueError("You can't specify both a end and a timespan") 

134 if end is None: 

135 if timespan > datetime.timedelta(minutes=0): 

136 end = begin + timespan # the normal case 

137 else: 

138 end = begin # the case where timespan is negative 

139 begin = begin + timespan # adding the negative to the start, i.e. subtracting it to bring back 

140 

141 assert begin is not None 

142 assert end is not None 

143 return begin, end 

144 

145 

146def getEfdData(client, topic, *, 

147 columns=None, 

148 prePadding=0, 

149 postPadding=0, 

150 dayObs=None, 

151 begin=None, 

152 end=None, 

153 timespan=None, 

154 event=None, 

155 expRecord=None, 

156 warn=True, 

157 ): 

158 """Get one or more EFD topics over a time range, synchronously. 

159 

160 The time range can be specified as either: 

161 * a dayObs, in which case the full 24 hour period is used, 

162 * a begin point and a end point, 

163 * a begin point and a timespan. 

164 * a mount event 

165 * an exposure record 

166 If it is desired to use an end time with a timespan, just specify it as the 

167 begin time and use a negative timespan. 

168 

169 The results from all topics are merged into a single dataframe. 

170 

171 Parameters 

172 ---------- 

173 client : `lsst_efd_client.efd_helper.EfdClient` 

174 The EFD client to use. 

175 topic : `str` 

176 The topic to query. 

177 columns : `list` of `str`, optional 

178 The columns to query. If not specified, all columns are queried. 

179 prePadding : `float` 

180 The amount of time before the nominal start of the query to include, in 

181 seconds. 

182 postPadding : `float` 

183 The amount of extra time after the nominal end of the query to include, 

184 in seconds. 

185 dayObs : `int`, optional 

186 The dayObs to query. If specified, this is used to determine the begin 

187 and end times. 

188 begin : `astropy.Time`, optional 

189 The begin time for the query. If specified, either a end time or a 

190 timespan must be supplied. 

191 end : `astropy.Time`, optional 

192 The end time for the query. If specified, a begin time must also be 

193 supplied. 

194 timespan : `astropy.TimeDelta`, optional 

195 The timespan for the query. If specified, a begin time must also be 

196 supplied. 

197 event : `lsst.summit.utils.efdUtils.TmaEvent`, optional 

198 The event to query. If specified, this is used to determine the begin 

199 and end times, and all other options are disallowed. 

200 expRecord : `lsst.daf.butler.dimensions.DimensionRecord`, optional 

201 The exposure record containing the timespan to query. If specified, all 

202 other options are disallowed. 

203 warn : bool, optional 

204 If ``True``, warn when no data is found. Exists so that utility code 

205 can disable warnings when checking for data, and therefore defaults to 

206 ``True``. 

207 

208 Returns 

209 ------- 

210 data : `pd.DataFrame` 

211 The merged data from all topics. 

212 

213 Raises 

214 ------ 

215 ValueError: 

216 If the topics are not in the EFD schema. 

217 ValueError: 

218 If both a dayObs and a begin/end or timespan are specified. 

219 ValueError: 

220 If a begin time is specified but no end time or timespan. 

221 

222 """ 

223 # TODO: DM-40100 ideally should calls mpts as necessary so that users 

224 # needn't care if things are packed 

225 

226 # supports aliases so that you can query with them. If there is no entry in 

227 # the alias dict then it queries with the supplied key. The fact the schema 

228 # is now being checked means this shouldn't be a problem now. 

229 

230 # TODO: RFC-948 Move this import back to top of file once is implemented. 

231 import nest_asyncio 

232 

233 begin, end = _getBeginEnd(dayObs, begin, end, timespan, event, expRecord) 

234 begin -= TimeDelta(prePadding, format='sec') 

235 end += TimeDelta(postPadding, format='sec') 

236 

237 nest_asyncio.apply() 

238 loop = asyncio.get_event_loop() 

239 ret = loop.run_until_complete(_getEfdData(client=client, 

240 topic=topic, 

241 begin=begin, 

242 end=end, 

243 columns=columns)) 

244 if ret.empty and warn: 

245 log = logging.getLogger(__name__) 

246 log.warning(f"Topic {topic} is in the schema, but no data was returned by the query for the specified" 

247 " time range") 

248 return ret 

249 

250 

251async def _getEfdData(client, topic, begin, end, columns=None): 

252 """Get data for a topic from the EFD over the specified time range. 

253 

254 Parameters 

255 ---------- 

256 client : `lsst_efd_client.efd_helper.EfdClient` 

257 The EFD client to use. 

258 topic : `str` 

259 The topic to query. 

260 begin : `astropy.Time` 

261 The begin time for the query. 

262 end : `astropy.Time` 

263 The end time for the query. 

264 columns : `list` of `str`, optional 

265 The columns to query. If not specified, all columns are returned. 

266 

267 Returns 

268 ------- 

269 data : `pd.DataFrame` 

270 The data from the query. 

271 """ 

272 if columns is None: 

273 columns = ['*'] 

274 

275 availableTopics = await client.get_topics() 

276 

277 if topic not in availableTopics: 

278 raise ValueError(f"Topic {topic} not in EFD schema") 

279 

280 data = await client.select_time_series(topic, columns, begin.utc, end.utc) 

281 

282 return data 

283 

284 

285def getMostRecentRowWithDataBefore(client, topic, timeToLookBefore, warnStaleAfterNMinutes=60*12): 

286 """Get the most recent row of data for a topic before a given time. 

287 

288 Parameters 

289 ---------- 

290 client : `lsst_efd_client.efd_helper.EfdClient` 

291 The EFD client to use. 

292 topic : `str` 

293 The topic to query. 

294 timeToLookBefore : `astropy.Time` 

295 The time to look before. 

296 warnStaleAfterNMinutes : `float`, optional 

297 The number of minutes after which to consider the data stale and issue 

298 a warning. 

299 

300 Returns 

301 ------- 

302 row : `pd.Series` 

303 The row of data from the EFD containing the most recent data before the 

304 specified time. 

305 

306 Raises 

307 ------ 

308 ValueError: 

309 If the topic is not in the EFD schema. 

310 """ 

311 staleAge = datetime.timedelta(warnStaleAfterNMinutes) 

312 

313 firstDayPossible = getDayObsStartTime(20190101) 

314 

315 if timeToLookBefore < firstDayPossible: 

316 raise ValueError(f"Requested time {timeToLookBefore} is before any data was put in the EFD") 

317 

318 df = pd.DataFrame() 

319 beginTime = timeToLookBefore 

320 while df.empty and beginTime > firstDayPossible: 

321 df = getEfdData(client, topic, begin=beginTime, timespan=-TIME_CHUNKING, warn=False) 

322 beginTime -= TIME_CHUNKING 

323 

324 if beginTime < firstDayPossible and df.empty: # we ran all the way back to the beginning of time 

325 raise ValueError(f"The entire EFD was searched backwards from {timeToLookBefore} and no data was " 

326 f"found in {topic=}") 

327 

328 lastRow = df.iloc[-1] 

329 commandTime = efdTimestampToAstropy(lastRow['private_efdStamp']) 

330 

331 commandAge = timeToLookBefore - commandTime 

332 if commandAge > staleAge: 

333 log = logging.getLogger(__name__) 

334 log.warning(f"Component {topic} was last set {commandAge.sec/60:.1} minutes" 

335 " before the requested time") 

336 

337 return lastRow 

338 

339 

340def makeEfdClient(testing=False): 

341 """Automatically create an EFD client based on the site. 

342 

343 Parameters 

344 ---------- 

345 testing : `bool`, optional 

346 Set to ``True`` if running in a test suite. This will default to using 

347 the USDF EFD, for which data has been recorded for replay by the ``vcr` 

348 package. Note data must be re-recorded to ``vcr`` from both inside and 

349 outside the USDF when the package/data changes, due to the use of a 

350 proxy meaning that the web requests are different depending on whether 

351 the EFD is being contacted from inside and outside the USDF. 

352 

353 Returns 

354 ------- 

355 efdClient : `lsst_efd_client.efd_helper.EfdClient` 

356 The EFD client to use for the current site. 

357 """ 

358 if not HAS_EFD_CLIENT: 

359 raise RuntimeError("Could not create EFD client because importing lsst_efd_client failed.") 

360 

361 if testing: 

362 return EfdClient('usdf_efd') 

363 

364 try: 

365 site = getSite() 

366 except ValueError as e: 

367 raise RuntimeError("Could not create EFD client as the site could not be determined") from e 

368 

369 if site == 'summit': 

370 return EfdClient('summit_efd') 

371 if site == 'tucson': 

372 return EfdClient('tucson_teststand_efd') 

373 if site == 'base': 

374 return EfdClient('base_efd') 

375 if site in ['staff-rsp', 'rubin-devl', 'usdf-k8s']: 

376 return EfdClient('usdf_efd') 

377 if site == 'jenkins': 

378 return EfdClient('usdf_efd') 

379 

380 raise RuntimeError(f"Could not create EFD client as the {site=} is not recognized") 

381 

382 

383def expRecordToTimespan(expRecord): 

384 """Get the timespan from an exposure record. 

385 

386 Returns the timespan in a format where it can be used to directly unpack 

387 into a efdClient.select_time_series() call. 

388 

389 Parameters 

390 ---------- 

391 expRecord : `lsst.daf.butler.dimensions.ExposureRecord` 

392 The exposure record. 

393 

394 Returns 

395 ------- 

396 timespanDict : `dict` 

397 The timespan in a format that can be used to directly unpack into a 

398 efdClient.select_time_series() call. 

399 """ 

400 return {'begin': expRecord.timespan.begin.utc, 

401 'end': expRecord.timespan.end.utc, 

402 } 

403 

404 

405def efdTimestampToAstropy(timestamp): 

406 """Get an efd timestamp as an astropy.time.Time object. 

407 

408 Parameters 

409 ---------- 

410 timestamp : `float` 

411 The timestamp, as a float. 

412 

413 Returns 

414 ------- 

415 time : `astropy.time.Time` 

416 The timestamp as an astropy.time.Time object. 

417 """ 

418 return Time(timestamp, format='unix') 

419 

420 

421def astropyToEfdTimestamp(time): 

422 """Get astropy Time object as an efd timestamp 

423 

424 Parameters 

425 ---------- 

426 time : `astropy.time.Time` 

427 The time as an astropy.time.Time object. 

428 

429 Returns 

430 ------- 

431 timestamp : `float` 

432 The timestamp, in UTC, in unix seconds. 

433 """ 

434 

435 return time.utc.unix 

436 

437 

438def clipDataToEvent(df, event): 

439 """Clip a padded dataframe to an event. 

440 

441 Parameters 

442 ---------- 

443 df : `pd.DataFrame` 

444 The dataframe to clip. 

445 event : `lsst.summit.utils.efdUtils.TmaEvent` 

446 The event to clip to. 

447 

448 Returns 

449 ------- 

450 clipped : `pd.DataFrame` 

451 The clipped dataframe. 

452 """ 

453 mask = (df['private_efdStamp'] >= event.begin.value) & (df['private_efdStamp'] <= event.end.value) 

454 clipped_df = df.loc[mask].copy() 

455 return clipped_df 

456 

457 

458def calcNextDay(dayObs): 

459 """Given an integer dayObs, calculate the next integer dayObs. 

460 

461 Integers are used for dayObs, but dayObs values are therefore not 

462 contiguous due to month/year ends etc, so this utility provides a robust 

463 way to get the integer dayObs which follows the one specified. 

464 

465 Parameters 

466 ---------- 

467 dayObs : `int` 

468 The dayObs, as an integer, e.g. 20231231 

469 

470 Returns 

471 ------- 

472 nextDayObs : `int` 

473 The next dayObs, as an integer, e.g. 20240101 

474 """ 

475 d1 = datetime.datetime.strptime(str(dayObs), '%Y%m%d') 

476 oneDay = datetime.timedelta(days=1) 

477 return int((d1 + oneDay).strftime('%Y%m%d')) 

478 

479 

480def getDayObsStartTime(dayObs): 

481 """Get the start of the given dayObs as an astropy.time.Time object. 

482 

483 The observatory rolls the date over at UTC-12. 

484 

485 Parameters 

486 ---------- 

487 dayObs : `int` 

488 The dayObs, as an integer, e.g. 20231225 

489 

490 Returns 

491 ------- 

492 time : `astropy.time.Time` 

493 The start of the dayObs as an astropy.time.Time object. 

494 """ 

495 pythonDateTime = datetime.datetime.strptime(str(dayObs), "%Y%m%d") 

496 return Time(pythonDateTime) + 12 * u.hour 

497 

498 

499def getDayObsEndTime(dayObs): 

500 """Get the end of the given dayObs as an astropy.time.Time object. 

501 

502 Parameters 

503 ---------- 

504 dayObs : `int` 

505 The dayObs, as an integer, e.g. 20231225 

506 

507 Returns 

508 ------- 

509 time : `astropy.time.Time` 

510 The end of the dayObs as an astropy.time.Time object. 

511 """ 

512 return getDayObsStartTime(dayObs) + 24 * u.hour 

513 

514 

515def getDayObsForTime(time): 

516 """Get the dayObs in which an astropy.time.Time object falls. 

517 

518 Parameters 

519 ---------- 

520 time : `astropy.time.Time` 

521 The time. 

522 

523 Returns 

524 ------- 

525 dayObs : `int` 

526 The dayObs, as an integer, e.g. 20231225 

527 """ 

528 twelveHours = datetime.timedelta(hours=-12) 

529 offset = TimeDelta(twelveHours, format='datetime') 

530 return int((time + offset).utc.isot[:10].replace('-', '')) 

531 

532 

533def getSubTopics(client, topic): 

534 """Get all the sub topics within a given topic. 

535 

536 Note that the topic need not be a complete one, for example, rather than 

537 doing `getSubTopics(client, 'lsst.sal.ATMCS')` to get all the topics for 

538 the AuxTel Mount Control System, you can do `getSubTopics(client, 

539 'lsst.sal.AT')` to get all which relate to the AuxTel in general. 

540 

541 Parameters 

542 ---------- 

543 client : `lsst_efd_client.efd_helper.EfdClient` 

544 The EFD client to use. 

545 topic : `str` 

546 The topic to query. 

547 

548 Returns 

549 ------- 

550 subTopics : `list` of `str` 

551 The sub topics. 

552 """ 

553 loop = asyncio.get_event_loop() 

554 topics = loop.run_until_complete(client.get_topics()) 

555 return sorted([t for t in topics if t.startswith(topic)])