Coverage for python/lsst/summit/utils/efdUtils.py: 16%

160 statements  

« prev     ^ index     » next       coverage.py v7.4.1, created at 2024-02-14 12:19 +0000

1# This file is part of summit_utils. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (https://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <https://www.gnu.org/licenses/>. 

21 

22import asyncio 

23from astropy.time import Time, TimeDelta 

24from astropy import units as u 

25import datetime 

26import logging 

27import pandas as pd 

28import re 

29from deprecated.sphinx import deprecated 

30 

31from lsst.utils.iteration import ensure_iterable 

32 

33from .utils import getSite 

34 

35HAS_EFD_CLIENT = True 

36try: 

37 from lsst_efd_client import EfdClient 

38except ImportError: 

39 HAS_EFD_CLIENT = False 

40 

41__all__ = [ 

42 'getEfdData', 

43 'getMostRecentRowWithDataBefore', 

44 'makeEfdClient', 

45 'expRecordToTimespan', 

46 'efdTimestampToAstropy', 

47 'astropyToEfdTimestamp', 

48 'clipDataToEvent', 

49 'calcNextDay', 

50 'getDayObsStartTime', 

51 'getDayObsEndTime', 

52 'getDayObsForTime', 

53 'getSubTopics', # deprecated, being removed in w_2023_50 

54 'getTopics', 

55] 

56 

57 

58COMMAND_ALIASES = { 

59 'raDecTarget': 'lsst.sal.MTPtg.command_raDecTarget', 

60 'moveToTarget': 'lsst.sal.MTMount.command_moveToTarget', 

61 'startTracking': 'lsst.sal.MTMount.command_startTracking', 

62 'stopTracking': 'lsst.sal.MTMount.command_stopTracking', 

63 'trackTarget': 'lsst.sal.MTMount.command_trackTarget', # issued at 20Hz - don't plot 

64} 

65 

66# When looking backwards in time to find the most recent state event, look back 

67# in chunks of this size. Too small, and there will be too many queries, too 

68# large and there will be too much data returned unnecessarily, as we only need 

69# one row by definition. Will tune this parameters in consultation with SQuaRE. 

70TIME_CHUNKING = datetime.timedelta(minutes=15) 

71 

72 

73def _getBeginEnd(dayObs=None, begin=None, end=None, timespan=None, event=None, expRecord=None): 

74 """Calculate the begin and end times to pass to _getEfdData, given the 

75 kwargs passed to getEfdData. 

76 

77 Parameters 

78 ---------- 

79 dayObs : `int` 

80 The dayObs to query. If specified, this is used to determine the begin 

81 and end times. 

82 begin : `astropy.Time` 

83 The begin time for the query. If specified, either an end time or a 

84 timespan must be supplied. 

85 end : `astropy.Time` 

86 The end time for the query. If specified, a begin time must also be 

87 supplied. 

88 timespan : `astropy.TimeDelta` 

89 The timespan for the query. If specified, a begin time must also be 

90 supplied. 

91 event : `lsst.summit.utils.efdUtils.TmaEvent` 

92 The event to query. If specified, this is used to determine the begin 

93 and end times, and all other options are disallowed. 

94 expRecord : `lsst.daf.butler.dimensions.DimensionRecord` 

95 The exposure record containing the timespan to query. If specified, all 

96 other options are disallowed. 

97 

98 Returns 

99 ------- 

100 begin : `astropy.Time` 

101 The begin time for the query. 

102 end : `astropy.Time` 

103 The end time for the query. 

104 """ 

105 if expRecord is not None: 

106 forbiddenOpts = [event, begin, end, timespan, dayObs] 

107 if any(x is not None for x in forbiddenOpts): 

108 raise ValueError("You can't specify both an expRecord and a begin/end or timespan or dayObs") 

109 begin = expRecord.timespan.begin 

110 end = expRecord.timespan.end 

111 return begin, end 

112 

113 if event is not None: 

114 forbiddenOpts = [begin, end, timespan, dayObs] 

115 if any(x is not None for x in forbiddenOpts): 

116 raise ValueError("You can't specify both an event and a begin/end or timespan or dayObs") 

117 begin = event.begin 

118 end = event.end 

119 return begin, end 

120 

121 # check for dayObs, and that other options aren't inconsistently specified 

122 if dayObs is not None: 

123 forbiddenOpts = [begin, end, timespan] 

124 if any(x is not None for x in forbiddenOpts): 

125 raise ValueError("You can't specify both a dayObs and a begin/end or timespan") 

126 begin = getDayObsStartTime(dayObs) 

127 end = getDayObsEndTime(dayObs) 

128 return begin, end 

129 # can now disregard dayObs entirely 

130 

131 if begin is None: 

132 raise ValueError("You must specify either a dayObs or a begin/end or begin/timespan") 

133 # can now rely on begin, so just need to deal with end/timespan 

134 

135 if end is None and timespan is None: 

136 raise ValueError("If you specify a begin, you must specify either a end or a timespan") 

137 if end is not None and timespan is not None: 

138 raise ValueError("You can't specify both a end and a timespan") 

139 if end is None: 

140 if timespan > datetime.timedelta(minutes=0): 

141 end = begin + timespan # the normal case 

142 else: 

143 end = begin # the case where timespan is negative 

144 begin = begin + timespan # adding the negative to the start, i.e. subtracting it to bring back 

145 

146 assert begin is not None 

147 assert end is not None 

148 return begin, end 

149 

150 

151def getEfdData(client, topic, *, 

152 columns=None, 

153 prePadding=0, 

154 postPadding=0, 

155 dayObs=None, 

156 begin=None, 

157 end=None, 

158 timespan=None, 

159 event=None, 

160 expRecord=None, 

161 warn=True, 

162 ): 

163 """Get one or more EFD topics over a time range, synchronously. 

164 

165 The time range can be specified as either: 

166 * a dayObs, in which case the full 24 hour period is used, 

167 * a begin point and a end point, 

168 * a begin point and a timespan. 

169 * a mount event 

170 * an exposure record 

171 If it is desired to use an end time with a timespan, just specify it as the 

172 begin time and use a negative timespan. 

173 

174 The results from all topics are merged into a single dataframe. 

175 

176 Parameters 

177 ---------- 

178 client : `lsst_efd_client.efd_helper.EfdClient` 

179 The EFD client to use. 

180 topic : `str` 

181 The topic to query. 

182 columns : `list` of `str`, optional 

183 The columns to query. If not specified, all columns are queried. 

184 prePadding : `float` 

185 The amount of time before the nominal start of the query to include, in 

186 seconds. 

187 postPadding : `float` 

188 The amount of extra time after the nominal end of the query to include, 

189 in seconds. 

190 dayObs : `int`, optional 

191 The dayObs to query. If specified, this is used to determine the begin 

192 and end times. 

193 begin : `astropy.Time`, optional 

194 The begin time for the query. If specified, either a end time or a 

195 timespan must be supplied. 

196 end : `astropy.Time`, optional 

197 The end time for the query. If specified, a begin time must also be 

198 supplied. 

199 timespan : `astropy.TimeDelta`, optional 

200 The timespan for the query. If specified, a begin time must also be 

201 supplied. 

202 event : `lsst.summit.utils.efdUtils.TmaEvent`, optional 

203 The event to query. If specified, this is used to determine the begin 

204 and end times, and all other options are disallowed. 

205 expRecord : `lsst.daf.butler.dimensions.DimensionRecord`, optional 

206 The exposure record containing the timespan to query. If specified, all 

207 other options are disallowed. 

208 warn : bool, optional 

209 If ``True``, warn when no data is found. Exists so that utility code 

210 can disable warnings when checking for data, and therefore defaults to 

211 ``True``. 

212 

213 Returns 

214 ------- 

215 data : `pd.DataFrame` 

216 The merged data from all topics. 

217 

218 Raises 

219 ------ 

220 ValueError: 

221 If the topics are not in the EFD schema. 

222 ValueError: 

223 If both a dayObs and a begin/end or timespan are specified. 

224 ValueError: 

225 If a begin time is specified but no end time or timespan. 

226 

227 """ 

228 # TODO: DM-40100 ideally should calls mpts as necessary so that users 

229 # needn't care if things are packed 

230 

231 # supports aliases so that you can query with them. If there is no entry in 

232 # the alias dict then it queries with the supplied key. The fact the schema 

233 # is now being checked means this shouldn't be a problem now. 

234 

235 # TODO: RFC-948 Move this import back to top of file once is implemented. 

236 import nest_asyncio 

237 

238 begin, end = _getBeginEnd(dayObs, begin, end, timespan, event, expRecord) 

239 begin -= TimeDelta(prePadding, format='sec') 

240 end += TimeDelta(postPadding, format='sec') 

241 

242 nest_asyncio.apply() 

243 loop = asyncio.get_event_loop() 

244 ret = loop.run_until_complete(_getEfdData(client=client, 

245 topic=topic, 

246 begin=begin, 

247 end=end, 

248 columns=columns)) 

249 if ret.empty and warn: 

250 log = logging.getLogger(__name__) 

251 log.warning(f"Topic {topic} is in the schema, but no data was returned by the query for the specified" 

252 " time range") 

253 return ret 

254 

255 

256async def _getEfdData(client, topic, begin, end, columns=None): 

257 """Get data for a topic from the EFD over the specified time range. 

258 

259 Parameters 

260 ---------- 

261 client : `lsst_efd_client.efd_helper.EfdClient` 

262 The EFD client to use. 

263 topic : `str` 

264 The topic to query. 

265 begin : `astropy.Time` 

266 The begin time for the query. 

267 end : `astropy.Time` 

268 The end time for the query. 

269 columns : `list` of `str`, optional 

270 The columns to query. If not specified, all columns are returned. 

271 

272 Returns 

273 ------- 

274 data : `pd.DataFrame` 

275 The data from the query. 

276 """ 

277 if columns is None: 

278 columns = ['*'] 

279 columns = list(ensure_iterable(columns)) 

280 

281 availableTopics = await client.get_topics() 

282 

283 if topic not in availableTopics: 

284 raise ValueError(f"Topic {topic} not in EFD schema") 

285 

286 data = await client.select_time_series(topic, columns, begin.utc, end.utc) 

287 

288 return data 

289 

290 

291def getMostRecentRowWithDataBefore(client, topic, timeToLookBefore, warnStaleAfterNMinutes=60*12): 

292 """Get the most recent row of data for a topic before a given time. 

293 

294 Parameters 

295 ---------- 

296 client : `lsst_efd_client.efd_helper.EfdClient` 

297 The EFD client to use. 

298 topic : `str` 

299 The topic to query. 

300 timeToLookBefore : `astropy.Time` 

301 The time to look before. 

302 warnStaleAfterNMinutes : `float`, optional 

303 The number of minutes after which to consider the data stale and issue 

304 a warning. 

305 

306 Returns 

307 ------- 

308 row : `pd.Series` 

309 The row of data from the EFD containing the most recent data before the 

310 specified time. 

311 

312 Raises 

313 ------ 

314 ValueError: 

315 If the topic is not in the EFD schema. 

316 """ 

317 staleAge = datetime.timedelta(warnStaleAfterNMinutes) 

318 

319 firstDayPossible = getDayObsStartTime(20190101) 

320 

321 if timeToLookBefore < firstDayPossible: 

322 raise ValueError(f"Requested time {timeToLookBefore} is before any data was put in the EFD") 

323 

324 df = pd.DataFrame() 

325 beginTime = timeToLookBefore 

326 while df.empty and beginTime > firstDayPossible: 

327 df = getEfdData(client, topic, begin=beginTime, timespan=-TIME_CHUNKING, warn=False) 

328 beginTime -= TIME_CHUNKING 

329 

330 if beginTime < firstDayPossible and df.empty: # we ran all the way back to the beginning of time 

331 raise ValueError(f"The entire EFD was searched backwards from {timeToLookBefore} and no data was " 

332 f"found in {topic=}") 

333 

334 lastRow = df.iloc[-1] 

335 commandTime = efdTimestampToAstropy(lastRow['private_efdStamp']) 

336 

337 commandAge = timeToLookBefore - commandTime 

338 if commandAge > staleAge: 

339 log = logging.getLogger(__name__) 

340 log.warning(f"Component {topic} was last set {commandAge.sec/60:.1} minutes" 

341 " before the requested time") 

342 

343 return lastRow 

344 

345 

346def makeEfdClient(testing=False): 

347 """Automatically create an EFD client based on the site. 

348 

349 Parameters 

350 ---------- 

351 testing : `bool`, optional 

352 Set to ``True`` if running in a test suite. This will default to using 

353 the USDF EFD, for which data has been recorded for replay by the ``vcr` 

354 package. Note data must be re-recorded to ``vcr`` from both inside and 

355 outside the USDF when the package/data changes, due to the use of a 

356 proxy meaning that the web requests are different depending on whether 

357 the EFD is being contacted from inside and outside the USDF. 

358 

359 Returns 

360 ------- 

361 efdClient : `lsst_efd_client.efd_helper.EfdClient` 

362 The EFD client to use for the current site. 

363 """ 

364 if not HAS_EFD_CLIENT: 

365 raise RuntimeError("Could not create EFD client because importing lsst_efd_client failed.") 

366 

367 if testing: 

368 return EfdClient('usdf_efd') 

369 

370 try: 

371 site = getSite() 

372 except ValueError as e: 

373 raise RuntimeError("Could not create EFD client as the site could not be determined") from e 

374 

375 if site == 'summit': 

376 return EfdClient('summit_efd') 

377 if site == 'tucson': 

378 return EfdClient('tucson_teststand_efd') 

379 if site == 'base': 

380 return EfdClient('base_efd') 

381 if site in ['staff-rsp', 'rubin-devl', 'usdf-k8s']: 

382 return EfdClient('usdf_efd') 

383 if site == 'jenkins': 

384 return EfdClient('usdf_efd') 

385 

386 raise RuntimeError(f"Could not create EFD client as the {site=} is not recognized") 

387 

388 

389def expRecordToTimespan(expRecord): 

390 """Get the timespan from an exposure record. 

391 

392 Returns the timespan in a format where it can be used to directly unpack 

393 into a efdClient.select_time_series() call. 

394 

395 Parameters 

396 ---------- 

397 expRecord : `lsst.daf.butler.dimensions.ExposureRecord` 

398 The exposure record. 

399 

400 Returns 

401 ------- 

402 timespanDict : `dict` 

403 The timespan in a format that can be used to directly unpack into a 

404 efdClient.select_time_series() call. 

405 """ 

406 return {'begin': expRecord.timespan.begin.utc, 

407 'end': expRecord.timespan.end.utc, 

408 } 

409 

410 

411def efdTimestampToAstropy(timestamp): 

412 """Get an efd timestamp as an astropy.time.Time object. 

413 

414 Parameters 

415 ---------- 

416 timestamp : `float` 

417 The timestamp, as a float. 

418 

419 Returns 

420 ------- 

421 time : `astropy.time.Time` 

422 The timestamp as an astropy.time.Time object. 

423 """ 

424 return Time(timestamp, format='unix') 

425 

426 

427def astropyToEfdTimestamp(time): 

428 """Get astropy Time object as an efd timestamp 

429 

430 Parameters 

431 ---------- 

432 time : `astropy.time.Time` 

433 The time as an astropy.time.Time object. 

434 

435 Returns 

436 ------- 

437 timestamp : `float` 

438 The timestamp, in UTC, in unix seconds. 

439 """ 

440 

441 return time.utc.unix 

442 

443 

444def clipDataToEvent(df, event, prePadding=0, postPadding=0, logger=None): 

445 """Clip a padded dataframe to an event. 

446 

447 Parameters 

448 ---------- 

449 df : `pd.DataFrame` 

450 The dataframe to clip. 

451 event : `lsst.summit.utils.efdUtils.TmaEvent` 

452 The event to clip to. 

453 prePadding : `float`, optional 

454 The amount of time before the nominal start of the event to include, in 

455 seconds. 

456 postPadding : `float`, optional 

457 The amount of extra time after the nominal end of the event to include, 

458 in seconds. 

459 logger : `logging.Logger`, optional 

460 The logger to use. If not specified, a new one is created. 

461 

462 Returns 

463 ------- 

464 clipped : `pd.DataFrame` 

465 The clipped dataframe. 

466 """ 

467 begin = event.begin.value - prePadding 

468 end = event.end.value + postPadding 

469 

470 if logger is None: 

471 logger = logging.getLogger(__name__) 

472 

473 if begin < df['private_efdStamp'].min(): 

474 logger.warning(f"Requested begin time {begin} is before the start of the data") 

475 if end > df['private_efdStamp'].max(): 

476 logger.warning(f"Requested end time {end} is after the end of the data") 

477 

478 mask = (df['private_efdStamp'] >= begin) & (df['private_efdStamp'] <= end) 

479 clipped_df = df.loc[mask].copy() 

480 return clipped_df 

481 

482 

483def calcNextDay(dayObs): 

484 """Given an integer dayObs, calculate the next integer dayObs. 

485 

486 Integers are used for dayObs, but dayObs values are therefore not 

487 contiguous due to month/year ends etc, so this utility provides a robust 

488 way to get the integer dayObs which follows the one specified. 

489 

490 Parameters 

491 ---------- 

492 dayObs : `int` 

493 The dayObs, as an integer, e.g. 20231231 

494 

495 Returns 

496 ------- 

497 nextDayObs : `int` 

498 The next dayObs, as an integer, e.g. 20240101 

499 """ 

500 d1 = datetime.datetime.strptime(str(dayObs), '%Y%m%d') 

501 oneDay = datetime.timedelta(days=1) 

502 return int((d1 + oneDay).strftime('%Y%m%d')) 

503 

504 

505def getDayObsStartTime(dayObs): 

506 """Get the start of the given dayObs as an astropy.time.Time object. 

507 

508 The observatory rolls the date over at UTC-12. 

509 

510 Parameters 

511 ---------- 

512 dayObs : `int` 

513 The dayObs, as an integer, e.g. 20231225 

514 

515 Returns 

516 ------- 

517 time : `astropy.time.Time` 

518 The start of the dayObs as an astropy.time.Time object. 

519 """ 

520 pythonDateTime = datetime.datetime.strptime(str(dayObs), "%Y%m%d") 

521 return Time(pythonDateTime) + 12 * u.hour 

522 

523 

524def getDayObsEndTime(dayObs): 

525 """Get the end of the given dayObs as an astropy.time.Time object. 

526 

527 Parameters 

528 ---------- 

529 dayObs : `int` 

530 The dayObs, as an integer, e.g. 20231225 

531 

532 Returns 

533 ------- 

534 time : `astropy.time.Time` 

535 The end of the dayObs as an astropy.time.Time object. 

536 """ 

537 return getDayObsStartTime(dayObs) + 24 * u.hour 

538 

539 

540def getDayObsForTime(time): 

541 """Get the dayObs in which an astropy.time.Time object falls. 

542 

543 Parameters 

544 ---------- 

545 time : `astropy.time.Time` 

546 The time. 

547 

548 Returns 

549 ------- 

550 dayObs : `int` 

551 The dayObs, as an integer, e.g. 20231225 

552 """ 

553 twelveHours = datetime.timedelta(hours=-12) 

554 offset = TimeDelta(twelveHours, format='datetime') 

555 return int((time + offset).utc.isot[:10].replace('-', '')) 

556 

557 

558@deprecated( 

559 reason="getSubTopics() has been replaced by getTopics() and using wildcards. " 

560 "Will be removed after w_2023_50.", 

561 version="w_2023_40", 

562 category=FutureWarning, 

563) 

564def getSubTopics(client, topic): 

565 """Get all the sub topics within a given topic. 

566 

567 Note that the topic need not be a complete one, for example, rather than 

568 doing `getSubTopics(client, 'lsst.sal.ATMCS')` to get all the topics for 

569 the AuxTel Mount Control System, you can do `getSubTopics(client, 

570 'lsst.sal.AT')` to get all which relate to the AuxTel in general. 

571 

572 Parameters 

573 ---------- 

574 client : `lsst_efd_client.efd_helper.EfdClient` 

575 The EFD client to use. 

576 topic : `str` 

577 The topic to query. 

578 

579 Returns 

580 ------- 

581 subTopics : `list` of `str` 

582 The sub topics. 

583 """ 

584 loop = asyncio.get_event_loop() 

585 topics = loop.run_until_complete(client.get_topics()) 

586 return sorted([t for t in topics if t.startswith(topic)]) 

587 

588 

589def getTopics(client, toFind, caseSensitive=False): 

590 """Return all the strings in topics which match the topic query string. 

591 

592 Supports wildcards, which are denoted as `*``, as per shell globs. 

593 

594 Example: 

595 >>> # assume topics are ['apple', 'banana', 'grape'] 

596 >>> getTopics(, 'a*p*') 

597 ['apple', 'grape'] 

598 

599 Parameters 

600 ---------- 

601 client : `lsst_efd_client.efd_helper.EfdClient` 

602 The EFD client to use. 

603 toFind : `str` 

604 The query string, with optional wildcards denoted as *. 

605 caseSensitive : `bool`, optional 

606 If ``True``, the query is case sensitive. Defaults to ``False``. 

607 

608 Returns 

609 ------- 

610 matches : `list` of `str` 

611 The list of matching topics. 

612 """ 

613 loop = asyncio.get_event_loop() 

614 topics = loop.run_until_complete(client.get_topics()) 

615 

616 # Replace wildcard with regex equivalent 

617 pattern = toFind.replace('*', '.*') 

618 flags = re.IGNORECASE if not caseSensitive else 0 

619 

620 matches = [] 

621 for topic in topics: 

622 if re.match(pattern, topic, flags): 

623 matches.append(topic) 

624 

625 return matches