Coverage for python/lsst/summit/utils/efdUtils.py: 16%

131 statements  

« prev     ^ index     » next       coverage.py v7.2.7, created at 2023-07-21 12:31 +0000

1# This file is part of summit_utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import asyncio 

23from astropy.time import Time, TimeDelta 

24from astropy import units as u 

25import datetime 

26import logging 

27import pandas as pd 

28 

29from .utils import getSite 

30 

31HAS_EFD_CLIENT = True 

32try: 

33 from lsst_efd_client import EfdClient 

34except ImportError: 

35 HAS_EFD_CLIENT = False 

36 

37__all__ = [ 

38 'getEfdData', 

39 'getMostRecentRowWithDataBefore', 

40 'makeEfdClient', 

41 'expRecordToTimespan', 

42 'efdTimestampToAstropy', 

43 'astropyToEfdTimestamp', 

44 'clipDataToEvent', 

45 'calcNextDay', 

46 'getDayObsStartTime', 

47 'getDayObsEndTime', 

48 'getDayObsForTime', 

49 'getSubTopics', 

50] 

51 

52 

53COMMAND_ALIASES = { 

54 'raDecTarget': 'lsst.sal.MTPtg.command_raDecTarget', 

55 'moveToTarget': 'lsst.sal.MTMount.command_moveToTarget', 

56 'startTracking': 'lsst.sal.MTMount.command_startTracking', 

57 'stopTracking': 'lsst.sal.MTMount.command_stopTracking', 

58 'trackTarget': 'lsst.sal.MTMount.command_trackTarget', # issued at 20Hz - don't plot 

59} 

60 

61# When looking backwards in time to find the most recent state event, look back 

62# in chunks of this size. Too small, and there will be too many queries, too 

63# large and there will be too much data returned unnecessarily, as we only need 

64# one row by definition. Will tune this parameters in consultation with SQuaRE. 

65TIME_CHUNKING = datetime.timedelta(minutes=15) 

66 

67 

68def _getBeginEnd(dayObs=None, begin=None, end=None, timespan=None, event=None, expRecord=None): 

69 """Calculate the begin and end times to pass to _getEfdData, given the 

70 kwargs passed to getEfdData. 

71 

72 Parameters 

73 ---------- 

74 dayObs : `int` 

75 The dayObs to query. If specified, this is used to determine the begin 

76 and end times. 

77 begin : `astropy.Time` 

78 The begin time for the query. If specified, either an end time or a 

79 timespan must be supplied. 

80 end : `astropy.Time` 

81 The end time for the query. If specified, a begin time must also be 

82 supplied. 

83 timespan : `astropy.TimeDelta` 

84 The timespan for the query. If specified, a begin time must also be 

85 supplied. 

86 event : `lsst.summit.utils.efdUtils.TmaEvent` 

87 The event to query. If specified, this is used to determine the begin 

88 and end times, and all other options are disallowed. 

89 expRecord : `lsst.daf.butler.dimensions.DimensionRecord` 

90 The exposure record containing the timespan to query. If specified, all 

91 other options are disallowed. 

92 

93 Returns 

94 ------- 

95 begin : `astropy.Time` 

96 The begin time for the query. 

97 end : `astropy.Time` 

98 The end time for the query. 

99 """ 

100 if expRecord is not None: 

101 forbiddenOpts = [event, begin, end, timespan, dayObs] 

102 if any(x is not None for x in forbiddenOpts): 

103 raise ValueError("You can't specify both an expRecord and a begin/end or timespan or dayObs") 

104 begin = expRecord.timespan.begin 

105 end = expRecord.timespan.end 

106 return begin, end 

107 

108 if event is not None: 

109 forbiddenOpts = [begin, end, timespan, dayObs] 

110 if any(x is not None for x in forbiddenOpts): 

111 raise ValueError("You can't specify both an event and a begin/end or timespan or dayObs") 

112 begin = event.begin 

113 end = event.end 

114 return begin, end 

115 

116 # check for dayObs, and that other options aren't inconsistently specified 

117 if dayObs is not None: 

118 forbiddenOpts = [begin, end, timespan] 

119 if any(x is not None for x in forbiddenOpts): 

120 raise ValueError("You can't specify both a dayObs and a begin/end or timespan") 

121 begin = getDayObsStartTime(dayObs) 

122 end = getDayObsEndTime(dayObs) 

123 return begin, end 

124 # can now disregard dayObs entirely 

125 

126 if begin is None: 

127 raise ValueError("You must specify either a dayObs or a begin/end or begin/timespan") 

128 # can now rely on begin, so just need to deal with end/timespan 

129 

130 if end is None and timespan is None: 

131 raise ValueError("If you specify a begin, you must specify either a end or a timespan") 

132 if end is not None and timespan is not None: 

133 raise ValueError("You can't specify both a end and a timespan") 

134 if end is None: 

135 if timespan > datetime.timedelta(minutes=0): 

136 end = begin + timespan # the normal case 

137 else: 

138 end = begin # the case where timespan is negative 

139 begin = begin + timespan # adding the negative to the start, i.e. subtracting it to bring back 

140 

141 assert begin is not None 

142 assert end is not None 

143 return begin, end 

144 

145 

146def getEfdData(client, topic, *, 

147 columns=None, 

148 prePadding=0, 

149 postPadding=0, 

150 dayObs=None, 

151 begin=None, 

152 end=None, 

153 timespan=None, 

154 event=None, 

155 expRecord=None, 

156 warn=True, 

157 ): 

158 """Get one or more EFD topics over a time range, synchronously. 

159 

160 The time range can be specified as either: 

161 * a dayObs, in which case the full 24 hour period is used, 

162 * a begin point and a end point, 

163 * a begin point and a timespan. 

164 * a mount event 

165 * an exposure record 

166 If it is desired to use an end time with a timespan, just specify it as the 

167 begin time and use a negative timespan. 

168 

169 The results from all topics are merged into a single dataframe. 

170 

171 Parameters 

172 ---------- 

173 client : `lsst_efd_client.efd_helper.EfdClient` 

174 The EFD client to use. 

175 topic : `str` 

176 The topic to query. 

177 columns : `list` of `str`, optional 

178 The columns to query. If not specified, all columns are queried. 

179 prePadding : `float` 

180 The amount of time before the nominal start of the query to include, in 

181 seconds. 

182 postPadding : `float` 

183 The amount of extra time after the nominal end of the query to include, 

184 in seconds. 

185 dayObs : `int`, optional 

186 The dayObs to query. If specified, this is used to determine the begin 

187 and end times. 

188 begin : `astropy.Time`, optional 

189 The begin time for the query. If specified, either a end time or a 

190 timespan must be supplied. 

191 end : `astropy.Time`, optional 

192 The end time for the query. If specified, a begin time must also be 

193 supplied. 

194 timespan : `astropy.TimeDelta`, optional 

195 The timespan for the query. If specified, a begin time must also be 

196 supplied. 

197 event : `lsst.summit.utils.efdUtils.TmaEvent`, optional 

198 The event to query. If specified, this is used to determine the begin 

199 and end times, and all other options are disallowed. 

200 expRecord : `lsst.daf.butler.dimensions.DimensionRecord`, optional 

201 The exposure record containing the timespan to query. If specified, all 

202 other options are disallowed. 

203 warn : bool, optional 

204 If ``True``, warn when no data is found. Exists so that utility code 

205 can disable warnings when checking for data, and therefore defaults to 

206 ``True``. 

207 

208 Returns 

209 ------- 

210 data : `pd.DataFrame` 

211 The merged data from all topics. 

212 

213 Raises 

214 ------ 

215 ValueError: 

216 If the topics are not in the EFD schema. 

217 ValueError: 

218 If both a dayObs and a begin/end or timespan are specified. 

219 ValueError: 

220 If a begin time is specified but no end time or timespan. 

221 

222 """ 

223 # TODO: DM-40100 ideally should calls mpts as necessary so that users 

224 # needn't care if things are packed 

225 

226 # supports aliases so that you can query with them. If there is no entry in 

227 # the alias dict then it queries with the supplied key. The fact the schema 

228 # is now being checked means this shouldn't be a problem now. 

229 

230 # TODO: RFC-948 Move this import back to top of file once is implemented. 

231 import nest_asyncio 

232 

233 begin, end = _getBeginEnd(dayObs, begin, end, timespan, event, expRecord) 

234 begin -= TimeDelta(prePadding, format='sec') 

235 end += TimeDelta(postPadding, format='sec') 

236 

237 nest_asyncio.apply() 

238 loop = asyncio.get_event_loop() 

239 ret = loop.run_until_complete(_getEfdData(client=client, 

240 topic=topic, 

241 begin=begin, 

242 end=end, 

243 columns=columns)) 

244 if ret.empty and warn: 

245 log = logging.getLogger(__name__) 

246 log.warning(f"Topic {topic} is in the schema, but no data was returned by the query for the specified" 

247 " time range") 

248 return ret 

249 

250 

251async def _getEfdData(client, topic, begin, end, columns=None): 

252 """Get data for a topic from the EFD over the specified time range. 

253 

254 Parameters 

255 ---------- 

256 client : `lsst_efd_client.efd_helper.EfdClient` 

257 The EFD client to use. 

258 topic : `str` 

259 The topic to query. 

260 begin : `astropy.Time` 

261 The begin time for the query. 

262 end : `astropy.Time` 

263 The end time for the query. 

264 columns : `list` of `str`, optional 

265 The columns to query. If not specified, all columns are returned. 

266 

267 Returns 

268 ------- 

269 data : `pd.DataFrame` 

270 The data from the query. 

271 """ 

272 if columns is None: 

273 columns = ['*'] 

274 

275 availableTopics = await client.get_topics() 

276 

277 if topic not in availableTopics: 

278 raise ValueError(f"Topic {topic} not in EFD schema") 

279 

280 data = await client.select_time_series(topic, columns, begin.utc, end.utc) 

281 

282 return data 

283 

284 

285def getMostRecentRowWithDataBefore(client, topic, timeToLookBefore, warnStaleAfterNMinutes=60*12): 

286 """Get the most recent row of data for a topic before a given time. 

287 

288 Parameters 

289 ---------- 

290 client : `lsst_efd_client.efd_helper.EfdClient` 

291 The EFD client to use. 

292 topic : `str` 

293 The topic to query. 

294 timeToLookBefore : `astropy.Time` 

295 The time to look before. 

296 warnStaleAfterNMinutes : `float`, optional 

297 The number of minutes after which to consider the data stale and issue 

298 a warning. 

299 

300 Returns 

301 ------- 

302 row : `pd.Series` 

303 The row of data from the EFD containing the most recent data before the 

304 specified time. 

305 

306 Raises 

307 ------ 

308 ValueError: 

309 If the topic is not in the EFD schema. 

310 """ 

311 staleAge = datetime.timedelta(warnStaleAfterNMinutes) 

312 

313 firstDayPossible = getDayObsStartTime(20190101) 

314 

315 if timeToLookBefore < firstDayPossible: 

316 raise ValueError(f"Requested time {timeToLookBefore} is before any data was put in the EFD") 

317 

318 df = pd.DataFrame() 

319 beginTime = timeToLookBefore 

320 while df.empty and beginTime > firstDayPossible: 

321 df = getEfdData(client, topic, begin=beginTime, timespan=-TIME_CHUNKING, warn=False) 

322 beginTime -= TIME_CHUNKING 

323 

324 if beginTime < firstDayPossible and df.empty: # we ran all the way back to the beginning of time 

325 raise ValueError(f"The entire EFD was searched backwards from {timeToLookBefore} and no data was " 

326 f"found in {topic=}") 

327 

328 lastRow = df.iloc[-1] 

329 commandTime = efdTimestampToAstropy(lastRow['private_efdStamp']) 

330 

331 commandAge = timeToLookBefore - commandTime 

332 if commandAge > staleAge: 

333 log = logging.getLogger(__name__) 

334 log.warning(f"Component {topic} was last set {commandAge.sec/60:.1} minutes" 

335 " before the requested time") 

336 

337 return lastRow 

338 

339 

340def makeEfdClient(): 

341 """Automatically create an EFD client based on the site. 

342 

343 Returns 

344 ------- 

345 efdClient : `lsst_efd_client.efd_helper.EfdClient` 

346 The EFD client to use for the current site. 

347 """ 

348 if not HAS_EFD_CLIENT: 

349 raise RuntimeError("Could not create EFD client because importing lsst_efd_client failed.") 

350 

351 try: 

352 site = getSite() 

353 except ValueError as e: 

354 raise RuntimeError("Could not create EFD client as the site could not be determined") from e 

355 

356 if site == 'summit': 

357 return EfdClient('summit_efd') 

358 if site == 'base': 

359 return EfdClient('summit_efd_copy') 

360 if site in ['staff-rsp', 'rubin-devl']: 

361 return EfdClient('usdf_efd') 

362 

363 raise RuntimeError(f"Could not create EFD client as the {site=} is not recognized") 

364 

365 

366def expRecordToTimespan(expRecord): 

367 """Get the timespan from an exposure record. 

368 

369 Returns the timespan in a format where it can be used to directly unpack 

370 into a efdClient.select_time_series() call. 

371 

372 Parameters 

373 ---------- 

374 expRecord : `lsst.daf.butler.dimensions.ExposureRecord` 

375 The exposure record. 

376 

377 Returns 

378 ------- 

379 timespanDict : `dict` 

380 The timespan in a format that can be used to directly unpack into a 

381 efdClient.select_time_series() call. 

382 """ 

383 return {'begin': expRecord.timespan.begin.utc, 

384 'end': expRecord.timespan.end.utc, 

385 } 

386 

387 

388def efdTimestampToAstropy(timestamp): 

389 """Get an efd timestamp as an astropy.time.Time object. 

390 

391 Parameters 

392 ---------- 

393 timestamp : `float` 

394 The timestamp, as a float. 

395 

396 Returns 

397 ------- 

398 time : `astropy.time.Time` 

399 The timestamp as an astropy.time.Time object. 

400 """ 

401 return Time(timestamp, format='unix') 

402 

403 

404def astropyToEfdTimestamp(time): 

405 """Get astropy Time object as an efd timestamp 

406 

407 Parameters 

408 ---------- 

409 time : `astropy.time.Time` 

410 The time as an astropy.time.Time object. 

411 

412 Returns 

413 ------- 

414 timestamp : `float` 

415 The timestamp, in UTC, in unix seconds. 

416 """ 

417 

418 return time.utc.unix 

419 

420 

421def clipDataToEvent(df, event): 

422 """Clip a padded dataframe to an event. 

423 

424 Parameters 

425 ---------- 

426 df : `pd.DataFrame` 

427 The dataframe to clip. 

428 event : `lsst.summit.utils.efdUtils.TmaEvent` 

429 The event to clip to. 

430 

431 Returns 

432 ------- 

433 clipped : `pd.DataFrame` 

434 The clipped dataframe. 

435 """ 

436 mask = (df['private_efdStamp'] >= event.begin.value) & (df['private_efdStamp'] <= event.end.value) 

437 clipped_df = df.loc[mask].copy() 

438 return clipped_df 

439 

440 

441def calcNextDay(dayObs): 

442 """Given an integer dayObs, calculate the next integer dayObs. 

443 

444 Integers are used for dayObs, but dayObs values are therefore not 

445 contiguous due to month/year ends etc, so this utility provides a robust 

446 way to get the integer dayObs which follows the one specified. 

447 

448 Parameters 

449 ---------- 

450 dayObs : `int` 

451 The dayObs, as an integer, e.g. 20231231 

452 

453 Returns 

454 ------- 

455 nextDayObs : `int` 

456 The next dayObs, as an integer, e.g. 20240101 

457 """ 

458 d1 = datetime.datetime.strptime(str(dayObs), '%Y%m%d') 

459 oneDay = datetime.timedelta(days=1) 

460 return int((d1 + oneDay).strftime('%Y%m%d')) 

461 

462 

463def getDayObsStartTime(dayObs): 

464 """Get the start of the given dayObs as an astropy.time.Time object. 

465 

466 The observatory rolls the date over at UTC-12. 

467 

468 Parameters 

469 ---------- 

470 dayObs : `int` 

471 The dayObs, as an integer, e.g. 20231225 

472 

473 Returns 

474 ------- 

475 time : `astropy.time.Time` 

476 The start of the dayObs as an astropy.time.Time object. 

477 """ 

478 pythonDateTime = datetime.datetime.strptime(str(dayObs), "%Y%m%d") 

479 return Time(pythonDateTime) + 12 * u.hour 

480 

481 

482def getDayObsEndTime(dayObs): 

483 """Get the end of the given dayObs as an astropy.time.Time object. 

484 

485 Parameters 

486 ---------- 

487 dayObs : `int` 

488 The dayObs, as an integer, e.g. 20231225 

489 

490 Returns 

491 ------- 

492 time : `astropy.time.Time` 

493 The end of the dayObs as an astropy.time.Time object. 

494 """ 

495 return getDayObsStartTime(dayObs) + 24 * u.hour 

496 

497 

498def getDayObsForTime(time): 

499 """Get the dayObs in which an astropy.time.Time object falls. 

500 

501 Parameters 

502 ---------- 

503 time : `astropy.time.Time` 

504 The time. 

505 

506 Returns 

507 ------- 

508 dayObs : `int` 

509 The dayObs, as an integer, e.g. 20231225 

510 """ 

511 twelveHours = datetime.timedelta(hours=-12) 

512 offset = TimeDelta(twelveHours, format='datetime') 

513 return int((time + offset).utc.isot[:10].replace('-', '')) 

514 

515 

516def getSubTopics(client, topic): 

517 """Get all the sub topics within a given topic. 

518 

519 Note that the topic need not be a complete one, for example, rather than 

520 doing `getSubTopics(client, 'lsst.sal.ATMCS')` to get all the topics for 

521 the AuxTel Mount Control System, you can do `getSubTopics(client, 

522 'lsst.sal.AT')` to get all which relate to the AuxTel in general. 

523 

524 Parameters 

525 ---------- 

526 client : `lsst_efd_client.efd_helper.EfdClient` 

527 The EFD client to use. 

528 topic : `str` 

529 The topic to query. 

530 

531 Returns 

532 ------- 

533 subTopics : `list` of `str` 

534 The sub topics. 

535 """ 

536 loop = asyncio.get_event_loop() 

537 topics = loop.run_until_complete(client.get_topics()) 

538 return sorted([t for t in topics if t.startswith(topic)])